mirror of
https://github.com/lukaszraczylo/graphql-monitoring-proxy.git
synced 2026-10-11 05:27:47 +00:00
fix: add HTTP backend readiness probes (#97)
This commit is contained in:
@@ -167,6 +167,7 @@ You can still use the non-prefixed environment variables in the spirit of the ba
|
||||
| `HOST_GRAPHQL` | The host to proxy the graphql endpoint | `http://localhost/` |
|
||||
| `HOST_GRAPHQL_READONLY` | The host to proxy the read-only graphql endpoint | `` |
|
||||
| `HEALTHCHECK_GRAPHQL_URL` | The URL to check the health of the graphql endpoint | `` |
|
||||
| `BACKEND_HEALTHCHECK_URL` | Optional HTTP readiness URL for startup and periodic backend checks; empty keeps the GraphQL probe | `` |
|
||||
| `JWT_USER_CLAIM_PATH` | Path to the user claim in the JWT token | `` |
|
||||
| `JWT_ROLE_CLAIM_PATH` | Path to the role claim in the JWT token | `` |
|
||||
| `JWT_VERIFY_SIGNATURE` | Verify the JWT signature before trusting its claims (see [Caching](#caching)) | `false` |
|
||||
@@ -1127,6 +1128,12 @@ With the `PURGE_METRICS_ON_CRAWL` enabled, the `graphql_proxy_requests_failed`,
|
||||
|
||||
If you'd like the `/healthz` endpoint to perform actual check for the connectivity to the graphql endpoint - set the `HEALTHCHECK_GRAPHQL_URL` environment variable to the exact URL of the graphql endpoint. The query executed will be `query { __typename }` and if the response is not `200 OK` - the healthcheck will fail. Remember that the endpoint is a full URL which you'd like to check, so it should include the protocol, host and path - for example `http://localhost:8080/v1/graphql` and it's NOT the same as value of `HOST_GRAPHQL` environment variable which should provide only the host, without path, ending with slash.
|
||||
|
||||
`BACKEND_HEALTHCHECK_URL` controls a separate check: the backend health manager's startup and periodic probes. Set it to an unauthenticated readiness endpoint, for example `http://127.0.0.1:8080/ready`. The manager sends `GET` to the supplied path and query, without a GraphQL body or authentication headers. Any 2xx response is healthy; network errors, timeouts, and other status codes are unhealthy. The request timeout remains five seconds, and redirects are not followed.
|
||||
|
||||
When this variable is empty, the manager continues to send an unauthenticated `POST` containing `{"query":"{__typename}"}` to `HOST_GRAPHQL`. It preserves a supplied path, or appends `/v1/graphql` when there is no path beyond `/`. A configured readiness URL must use HTTP or HTTPS with a nonempty host and no userinfo or fragment. Invalid configuration stops startup; it does not fall back to GraphQL. Do not put secrets in the URL: probe targets are logged.
|
||||
|
||||
This setting does not change `HEALTHCHECK_GRAPHQL_URL`, the API `/healthz` check, GraphQL authorization, or headers on forwarded client requests. Keep the GraphQL endpoint protected; only the readiness endpoint must accept these unauthenticated probes.
|
||||
|
||||
#### Monitoring endpoint
|
||||
|
||||
Example metrics produced by the proxy:
|
||||
|
||||
+2
-2
@@ -62,7 +62,7 @@ func TestApiBackendHealth_HealthyManager_Returns200(t *testing.T) {
|
||||
defer func() { backendHealthManager = orig }()
|
||||
|
||||
// inject a healthy manager directly (bypassing sync.Once)
|
||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", libpack_logger.New())
|
||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", "", libpack_logger.New())
|
||||
mgr.isHealthy.Store(true)
|
||||
backendHealthManager = mgr
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestApiBackendHealth_UnhealthyManager_Returns503(t *testing.T) {
|
||||
orig := backendHealthManager
|
||||
defer func() { backendHealthManager = orig }()
|
||||
|
||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", libpack_logger.New())
|
||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", "", libpack_logger.New())
|
||||
mgr.isHealthy.Store(false)
|
||||
backendHealthManager = mgr
|
||||
|
||||
|
||||
+64
-49
@@ -22,6 +22,8 @@ type BackendHealthManager struct {
|
||||
logger *libpack_logger.Logger
|
||||
cancel context.CancelFunc
|
||||
backendURL string
|
||||
healthCheckURL string
|
||||
useHTTPReadiness bool
|
||||
checkInterval time.Duration
|
||||
maxRetries int
|
||||
mu sync.RWMutex
|
||||
@@ -32,18 +34,41 @@ type BackendHealthManager struct {
|
||||
}
|
||||
|
||||
// NewBackendHealthManager creates a new backend health manager
|
||||
func NewBackendHealthManager(client *fasthttp.Client, backendURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||
func NewBackendHealthManager(client *fasthttp.Client, backendURL, readinessURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
healthCheckURL := readinessURL
|
||||
if healthCheckURL == "" {
|
||||
healthCheckURL = backendGraphQLHealthURL(backendURL)
|
||||
} else {
|
||||
// Path normalization is a client setting in fasthttp. Use a separate
|
||||
// pool so readiness URLs stay exact without changing forwarded traffic.
|
||||
client = &fasthttp.Client{
|
||||
Dial: client.Dial,
|
||||
DialTimeout: client.DialTimeout,
|
||||
TLSConfig: client.TLSConfig,
|
||||
ReadTimeout: client.ReadTimeout,
|
||||
WriteTimeout: client.WriteTimeout,
|
||||
MaxIdleConnDuration: client.MaxIdleConnDuration,
|
||||
MaxConnDuration: client.MaxConnDuration,
|
||||
ReadBufferSize: client.ReadBufferSize,
|
||||
WriteBufferSize: client.WriteBufferSize,
|
||||
MaxResponseBodySize: client.MaxResponseBodySize,
|
||||
NoDefaultUserAgentHeader: client.NoDefaultUserAgentHeader,
|
||||
DisablePathNormalizing: true,
|
||||
}
|
||||
}
|
||||
return &BackendHealthManager{
|
||||
client: client,
|
||||
backendURL: backendURL,
|
||||
checkInterval: 5 * time.Second,
|
||||
maxRetries: 30, // 30 * 5s = 2.5 minutes max startup wait
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
logger: logger,
|
||||
startupProbe: true,
|
||||
readinessChan: make(chan bool, 1),
|
||||
client: client,
|
||||
backendURL: backendURL,
|
||||
healthCheckURL: healthCheckURL,
|
||||
useHTTPReadiness: readinessURL != "",
|
||||
checkInterval: 5 * time.Second,
|
||||
maxRetries: 30, // 30 * 5s = 2.5 minutes max startup wait
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
logger: logger,
|
||||
startupProbe: true,
|
||||
readinessChan: make(chan bool, 1),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +84,7 @@ func (bhm *BackendHealthManager) WaitForBackendReady(timeout time.Duration) erro
|
||||
Message: "Waiting for GraphQL backend to become ready",
|
||||
Pairs: map[string]any{
|
||||
"backend_url": bhm.backendURL,
|
||||
"check_url": bhm.healthCheckURL,
|
||||
"timeout": timeout.String(),
|
||||
},
|
||||
})
|
||||
@@ -149,43 +175,15 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
||||
defer fasthttp.ReleaseRequest(req)
|
||||
defer fasthttp.ReleaseResponse(resp)
|
||||
|
||||
// Determine the health check URL
|
||||
// If backendURL is just "http://host:port" or "http://host:port/", append /v1/graphql
|
||||
// If it has a path like "/v1/graphql", use that path
|
||||
healthCheckURL := bhm.backendURL
|
||||
hasGraphQLPath := false
|
||||
|
||||
if len(bhm.backendURL) > 0 {
|
||||
// Simple check: if URL has a path component beyond just "/"
|
||||
lastSlash := -1
|
||||
protoEnd := 0
|
||||
if idx := strings.Index(bhm.backendURL, "://"); idx >= 0 {
|
||||
protoEnd = idx + 3
|
||||
}
|
||||
for i := protoEnd; i < len(bhm.backendURL); i++ {
|
||||
if bhm.backendURL[i] == '/' {
|
||||
lastSlash = i
|
||||
break
|
||||
}
|
||||
}
|
||||
// Has path if there's a slash after protocol and it's not the last char or followed by more path
|
||||
hasGraphQLPath = lastSlash >= protoEnd && lastSlash < len(bhm.backendURL)-1
|
||||
|
||||
// If no GraphQL path, append /v1/graphql (standard Hasura endpoint)
|
||||
if !hasGraphQLPath {
|
||||
// Remove trailing slash if present
|
||||
baseURL := strings.TrimSuffix(bhm.backendURL, "/")
|
||||
healthCheckURL = baseURL + "/v1/graphql"
|
||||
}
|
||||
req.SetRequestURI(bhm.healthCheckURL)
|
||||
if bhm.useHTTPReadiness {
|
||||
req.Header.SetMethod(http.MethodGet)
|
||||
} else {
|
||||
req.Header.SetMethod(http.MethodPost)
|
||||
req.Header.SetContentType("application/json")
|
||||
req.SetBodyString(`{"query":"{__typename}"}`)
|
||||
}
|
||||
|
||||
// Always send GraphQL introspection query for health check
|
||||
healthQuery := `{"query":"{__typename}"}`
|
||||
req.SetRequestURI(healthCheckURL)
|
||||
req.Header.SetMethod(http.MethodPost)
|
||||
req.Header.SetContentType("application/json")
|
||||
req.SetBody([]byte(healthQuery))
|
||||
|
||||
// Short timeout for health checks
|
||||
err := bhm.client.DoTimeout(req, resp, 5*time.Second)
|
||||
if err != nil {
|
||||
@@ -193,7 +191,7 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
||||
Message: "Backend health check failed",
|
||||
Pairs: map[string]any{
|
||||
"error": err.Error(),
|
||||
"check_url": healthCheckURL,
|
||||
"check_url": bhm.healthCheckURL,
|
||||
},
|
||||
})
|
||||
return false
|
||||
@@ -207,7 +205,7 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
||||
Message: "Backend returned unhealthy status",
|
||||
Pairs: map[string]any{
|
||||
"status_code": statusCode,
|
||||
"check_url": healthCheckURL,
|
||||
"check_url": bhm.healthCheckURL,
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -284,6 +282,9 @@ func (bhm *BackendHealthManager) Shutdown() {
|
||||
return
|
||||
}
|
||||
bhm.cancel()
|
||||
if bhm.useHTTPReadiness {
|
||||
bhm.client.CloseIdleConnections()
|
||||
}
|
||||
if bhm.logger != nil {
|
||||
bhm.logger.Info(&libpack_logger.LogMessage{
|
||||
Message: "Backend health manager shut down",
|
||||
@@ -298,9 +299,9 @@ var (
|
||||
)
|
||||
|
||||
// InitializeBackendHealth initializes the backend health manager
|
||||
func InitializeBackendHealth(client *fasthttp.Client, backendURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||
func InitializeBackendHealth(client *fasthttp.Client, backendURL, readinessURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||
backendHealthOnce.Do(func() {
|
||||
backendHealthManager = NewBackendHealthManager(client, backendURL, logger)
|
||||
backendHealthManager = NewBackendHealthManager(client, backendURL, readinessURL, logger)
|
||||
})
|
||||
return backendHealthManager
|
||||
}
|
||||
@@ -309,3 +310,17 @@ func InitializeBackendHealth(client *fasthttp.Client, backendURL string, logger
|
||||
func GetBackendHealthManager() *BackendHealthManager {
|
||||
return backendHealthManager
|
||||
}
|
||||
|
||||
func backendGraphQLHealthURL(backendURL string) string {
|
||||
if backendURL == "" {
|
||||
return ""
|
||||
}
|
||||
protoEnd := 0
|
||||
if idx := strings.Index(backendURL, "://"); idx >= 0 {
|
||||
protoEnd = idx + 3
|
||||
}
|
||||
if pathStart := strings.IndexByte(backendURL[protoEnd:], '/'); pathStart >= 0 && protoEnd+pathStart < len(backendURL)-1 {
|
||||
return backendURL
|
||||
}
|
||||
return strings.TrimSuffix(backendURL, "/") + "/v1/graphql"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
libpack_logger "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
func TestBackendReadinessProtectedGraphQL(t *testing.T) {
|
||||
var readyStatus atomic.Int32
|
||||
readyStatus.Store(http.StatusOK)
|
||||
var graphqlCalls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/v1/graphql":
|
||||
graphqlCalls.Add(1)
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
case "/ready":
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
assert.Empty(t, r.Header.Get("X-Husaria-Admin-Secret"))
|
||||
assert.Empty(t, r.Header.Get("X-Hasura-Admin-Secret"))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.Empty(t, body)
|
||||
w.WriteHeader(int(readyStatus.Load()))
|
||||
default:
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
client := &fasthttp.Client{}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
logger := libpack_logger.New().SetOutput(io.Discard)
|
||||
legacy := NewBackendHealthManager(client, server.URL, "", logger)
|
||||
t.Cleanup(legacy.Shutdown)
|
||||
require.False(t, legacy.checkBackendHealth(), "GraphQL must reject the unauthenticated probe")
|
||||
|
||||
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", logger)
|
||||
t.Cleanup(manager.Shutdown)
|
||||
manager.checkInterval = 10 * time.Millisecond
|
||||
manager.StartHealthChecking()
|
||||
require.NoError(t, manager.WaitForBackendReady(time.Second))
|
||||
require.True(t, manager.IsHealthy())
|
||||
|
||||
readyStatus.Store(http.StatusServiceUnavailable)
|
||||
require.Eventually(t, func() bool { return !manager.IsHealthy() }, time.Second, 5*time.Millisecond)
|
||||
readyStatus.Store(http.StatusOK)
|
||||
require.Eventually(t, manager.IsHealthy, time.Second, 5*time.Millisecond)
|
||||
assert.Equal(t, int32(1), graphqlCalls.Load(), "readiness checks must not query protected GraphQL")
|
||||
require.False(t, legacy.checkBackendHealth(), "readiness must not bypass GraphQL authorization")
|
||||
}
|
||||
|
||||
func TestBackendReadinessExactTarget(t *testing.T) {
|
||||
const target = "/checks//ready%2Fdatabase?probe=one%2Ftwo&probe=three"
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, target, r.RequestURI)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
client := &fasthttp.Client{}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
manager := NewBackendHealthManager(client, "http://unused.invalid", server.URL+target, libpack_logger.New().SetOutput(io.Discard))
|
||||
t.Cleanup(manager.Shutdown)
|
||||
assert.True(t, manager.checkBackendHealth())
|
||||
}
|
||||
|
||||
func TestBackendReadinessStatus(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
status int
|
||||
healthy bool
|
||||
}{
|
||||
{name: "good OK", status: http.StatusOK, healthy: true},
|
||||
{name: "good no content", status: http.StatusNoContent, healthy: true},
|
||||
{name: "edge final success", status: 299, healthy: true},
|
||||
{name: "bad redirect", status: http.StatusFound},
|
||||
{name: "bad unauthorized", status: http.StatusUnauthorized},
|
||||
{name: "bad unavailable", status: http.StatusServiceUnavailable},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/ready" {
|
||||
t.Error("readiness probe followed a redirect")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/redirected")
|
||||
w.WriteHeader(tc.status)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
client := &fasthttp.Client{}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||
t.Cleanup(manager.Shutdown)
|
||||
assert.Equal(t, tc.healthy, manager.checkBackendHealth())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendReadinessNetworkFailure(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
server.Close()
|
||||
client := &fasthttp.Client{}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||
t.Cleanup(manager.Shutdown)
|
||||
assert.False(t, manager.checkBackendHealth())
|
||||
}
|
||||
|
||||
func TestBackendReadinessTimeout(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
<-release
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
t.Cleanup(func() { close(release) })
|
||||
client := &fasthttp.Client{ReadTimeout: 20 * time.Millisecond}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||
t.Cleanup(manager.Shutdown)
|
||||
assert.False(t, manager.checkBackendHealth())
|
||||
}
|
||||
|
||||
func TestBackendReadinessLegacyGraphQL(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
configuredPath string
|
||||
requestURI string
|
||||
}{
|
||||
{name: "good bare host", requestURI: "/v1/graphql"},
|
||||
{name: "edge root path", configuredPath: "/", requestURI: "/v1/graphql"},
|
||||
{name: "good custom path and query", configuredPath: "/graphql?health=1", requestURI: "/graphql?health=1"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodPost, r.Method)
|
||||
assert.Equal(t, tc.requestURI, r.RequestURI)
|
||||
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
body, err := io.ReadAll(r.Body)
|
||||
assert.NoError(t, err)
|
||||
assert.JSONEq(t, `{"query":"{__typename}"}`, string(body))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
client := &fasthttp.Client{}
|
||||
t.Cleanup(client.CloseIdleConnections)
|
||||
manager := NewBackendHealthManager(client, server.URL+tc.configuredPath, "", libpack_logger.New().SetOutput(io.Discard))
|
||||
t.Cleanup(manager.Shutdown)
|
||||
assert.True(t, manager.checkBackendHealth())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readinessURLWithUserinfo(user *url.Userinfo) string {
|
||||
u := url.URL{Scheme: "http", Host: "example.test", Path: "/ready", User: user}
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func TestBackendReadinessURLValidation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
url string
|
||||
valid bool
|
||||
}{
|
||||
{name: "edge unset", valid: true},
|
||||
{name: "good HTTP", url: "http://127.0.0.1:8080/ready", valid: true},
|
||||
{name: "good HTTPS query", url: "https://example.test/check?database=primary", valid: true},
|
||||
{name: "good IPv6", url: "http://[::1]:8080/ready", valid: true},
|
||||
{name: "bad relative", url: "/ready"},
|
||||
{name: "bad scheme", url: "ftp://example.test/ready"},
|
||||
{name: "bad missing host", url: "http:///ready"},
|
||||
{name: "bad port only host", url: "http://:8080/ready"},
|
||||
{name: "bad opaque", url: "http:ready"},
|
||||
{name: "bad userinfo", url: readinessURLWithUserinfo(url.UserPassword("operator", "secret"))},
|
||||
{name: "bad user only", url: readinessURLWithUserinfo(url.User("operator"))},
|
||||
{name: "bad escaped userinfo", url: strings.Replace(readinessURLWithUserinfo(url.UserPassword("operator", "%zz")), "%25zz", "%zz", 1)},
|
||||
{name: "bad path escape", url: "http://example.test/%zz"},
|
||||
{name: "bad port", url: "http://example.test:invalid/ready"},
|
||||
{name: "bad port range", url: "http://example.test:65536/ready"},
|
||||
{name: "bad fragment", url: "http://example.test/ready#ignored"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := validateBackendHealthcheckURL(tc.url)
|
||||
if tc.valid {
|
||||
require.NoError(t, err)
|
||||
} else {
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), tc.url)
|
||||
assert.NotContains(t, err.Error(), "operator")
|
||||
assert.NotContains(t, err.Error(), "secret")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendReadinessInvalidConfigStopsStartup(t *testing.T) {
|
||||
const helper = "GMP_TEST_INVALID_READINESS_CONFIG"
|
||||
if os.Getenv(helper) == "1" {
|
||||
parseConfig()
|
||||
t.Fatal("invalid readiness configuration did not stop startup")
|
||||
}
|
||||
command := exec.Command(os.Args[0], "-test.run=^TestBackendReadinessInvalidConfigStopsStartup$")
|
||||
for _, variable := range os.Environ() {
|
||||
if !strings.HasPrefix(variable, "GMP_BACKEND_HEALTHCHECK_URL=") && !strings.HasPrefix(variable, helper+"=") {
|
||||
command.Env = append(command.Env, variable)
|
||||
}
|
||||
}
|
||||
command.Env = append(command.Env, helper+"=1", "GMP_BACKEND_HEALTHCHECK_URL="+readinessURLWithUserinfo(url.UserPassword("operator", "secret")))
|
||||
output, err := command.CombinedOutput()
|
||||
var exitError *exec.ExitError
|
||||
require.ErrorAs(t, err, &exitError)
|
||||
assert.Equal(t, 1, exitError.ExitCode())
|
||||
assert.Contains(t, string(output), "Invalid BACKEND_HEALTHCHECK_URL")
|
||||
assert.NotContains(t, string(output), "operator")
|
||||
assert.NotContains(t, string(output), "secret")
|
||||
}
|
||||
@@ -77,16 +77,8 @@ func (suite *ConnectionResilienceTestSuite) TearDownTest() {
|
||||
|
||||
// TestBackendHealthManager tests the backend health monitoring
|
||||
func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
||||
suite.Run("initialization", func() {
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
suite.NotNil(healthMgr)
|
||||
suite.Equal(cfg.Server.HostGraphQL, healthMgr.backendURL)
|
||||
suite.Equal(5*time.Second, healthMgr.checkInterval)
|
||||
suite.Equal(30, healthMgr.maxRetries)
|
||||
})
|
||||
|
||||
suite.Run("health check success", func() {
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||
isHealthy := healthMgr.checkBackendHealth()
|
||||
suite.True(isHealthy)
|
||||
suite.GreaterOrEqual(suite.mockServerCalled.Load(), int32(1))
|
||||
@@ -94,13 +86,13 @@ func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
||||
|
||||
suite.Run("health check failure", func() {
|
||||
// Use invalid URL to simulate failure
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||
isHealthy := healthMgr.checkBackendHealth()
|
||||
suite.False(isHealthy)
|
||||
})
|
||||
|
||||
suite.Run("startup readiness with healthy backend", func() {
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||
err := healthMgr.WaitForBackendReady(10 * time.Second)
|
||||
suite.NoError(err)
|
||||
suite.True(healthMgr.IsHealthy())
|
||||
@@ -108,7 +100,7 @@ func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
||||
|
||||
suite.Run("startup readiness timeout", func() {
|
||||
// Use invalid URL to simulate backend not ready
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||
err := healthMgr.WaitForBackendReady(2 * time.Second)
|
||||
suite.Error(err)
|
||||
suite.Contains(err.Error(), "did not become ready")
|
||||
@@ -177,7 +169,7 @@ func (suite *ConnectionResilienceTestSuite) TestConnectionPoolManager() {
|
||||
func (suite *ConnectionResilienceTestSuite) TestIntegratedHealthManagement() {
|
||||
suite.Run("global initialization", func() {
|
||||
// Initialize global instances
|
||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||
poolMgr := NewConnectionPoolManager(cfg.Client.FastProxyClient)
|
||||
|
||||
// Set global instances
|
||||
@@ -194,7 +186,7 @@ func (suite *ConnectionResilienceTestSuite) TestIntegratedHealthManagement() {
|
||||
// and may have already fired earlier in the process (e.g. via parseConfig in
|
||||
// another test), in which case it returns whatever the global currently is —
|
||||
// which TearDownTest above just nilled.
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||
backendHealthManager = healthMgr
|
||||
|
||||
// Start health checking
|
||||
@@ -272,7 +264,7 @@ func TestConnectionResilienceSuite(t *testing.T) {
|
||||
// periodic checking — otherwise the backend could never be marked healthy
|
||||
// again even after it recovers.
|
||||
func (suite *ConnectionResilienceTestSuite) TestHealthChecksStartAfterStartupTimeout() {
|
||||
bhm := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
||||
bhm := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||
err := bhm.WaitForBackendReady(200 * time.Millisecond)
|
||||
suite.Error(err, "expected timeout for unreachable backend")
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
||||
client := &fasthttp.Client{}
|
||||
|
||||
t.Run("updateHealthStatus healthy→unhealthy transition", func(t *testing.T) {
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||
defer bhm.Shutdown()
|
||||
|
||||
// Start healthy
|
||||
@@ -203,7 +203,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("updateHealthStatus unhealthy→healthy resets counter", func(t *testing.T) {
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||
defer bhm.Shutdown()
|
||||
|
||||
bhm.isHealthy.Store(false)
|
||||
@@ -219,7 +219,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("GetLastHealthCheck round-trip", func(t *testing.T) {
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||
defer bhm.Shutdown()
|
||||
|
||||
before := time.Now()
|
||||
|
||||
@@ -246,6 +246,33 @@ func validateJWTClaimPath(path string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateBackendHealthcheckURL(rawURL string) error {
|
||||
if rawURL == "" {
|
||||
return nil
|
||||
}
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
// Parse errors can include credentials from the rejected URL.
|
||||
return fmt.Errorf("must be a valid HTTP or HTTPS URL")
|
||||
}
|
||||
if (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.Opaque != "" {
|
||||
return fmt.Errorf("must use HTTP or HTTPS with a nonempty host")
|
||||
}
|
||||
if u.User != nil {
|
||||
return fmt.Errorf("must not contain userinfo or credentials")
|
||||
}
|
||||
if u.Fragment != "" {
|
||||
return fmt.Errorf("must not contain a fragment")
|
||||
}
|
||||
if port := u.Port(); port != "" {
|
||||
n, err := strconv.Atoi(port)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return fmt.Errorf("must use a port between 1 and 65535")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseConfig loads and parses the configuration.
|
||||
func parseConfig() {
|
||||
libpack_config.PKG_NAME = "graphql_proxy"
|
||||
@@ -259,6 +286,11 @@ func parseConfig() {
|
||||
c.Server.BindAddress = getDetailsFromEnv("BIND_ADDRESS", "")
|
||||
c.Server.HostGraphQL = getDetailsFromEnv("HOST_GRAPHQL", "http://localhost/")
|
||||
c.Server.HostGraphQLReadOnly = getDetailsFromEnv("HOST_GRAPHQL_READONLY", "")
|
||||
c.Server.BackendHealthcheckURL = getDetailsFromEnv("BACKEND_HEALTHCHECK_URL", "")
|
||||
if err := validateBackendHealthcheckURL(c.Server.BackendHealthcheckURL); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "CRITICAL ERROR: Invalid BACKEND_HEALTHCHECK_URL: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
// Client configurations
|
||||
c.Client.JWTUserClaimPath = getDetailsFromEnv("JWT_USER_CLAIM_PATH", "")
|
||||
c.Client.JWTRoleClaimPath = getDetailsFromEnv("JWT_ROLE_CLAIM_PATH", "")
|
||||
@@ -678,7 +710,7 @@ func parseConfig() {
|
||||
|
||||
// Initialize backend health manager
|
||||
if cfg.Server.HostGraphQL != "" {
|
||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Server.BackendHealthcheckURL, cfg.Logger)
|
||||
// Start health checking in background
|
||||
healthMgr.StartHealthChecking()
|
||||
}
|
||||
|
||||
+5
-4
@@ -64,10 +64,11 @@ type config struct {
|
||||
DisableTLSVerify bool
|
||||
}
|
||||
Server struct {
|
||||
HostGraphQL string
|
||||
HostGraphQLReadOnly string
|
||||
HealthcheckGraphQL string
|
||||
AllowURLs []string // List of allowed URL paths for access control
|
||||
HostGraphQL string
|
||||
HostGraphQLReadOnly string
|
||||
HealthcheckGraphQL string
|
||||
BackendHealthcheckURL string
|
||||
AllowURLs []string // List of allowed URL paths for access control
|
||||
// BindAddress is the host portion of every listener's bind address
|
||||
// (GraphQL proxy, admin API, monitoring). Empty (the default)
|
||||
// preserves the existing ":port" behavior of binding all
|
||||
|
||||
Reference in New Issue
Block a user