mirror of
https://github.com/lukaszraczylo/graphql-monitoring-proxy.git
synced 2026-10-12 05:32:36 +00:00
fix: add HTTP backend readiness probes (#97)
This commit is contained in:
@@ -167,6 +167,7 @@ You can still use the non-prefixed environment variables in the spirit of the ba
|
|||||||
| `HOST_GRAPHQL` | The host to proxy the graphql endpoint | `http://localhost/` |
|
| `HOST_GRAPHQL` | The host to proxy the graphql endpoint | `http://localhost/` |
|
||||||
| `HOST_GRAPHQL_READONLY` | The host to proxy the read-only graphql endpoint | `` |
|
| `HOST_GRAPHQL_READONLY` | The host to proxy the read-only graphql endpoint | `` |
|
||||||
| `HEALTHCHECK_GRAPHQL_URL` | The URL to check the health of the graphql endpoint | `` |
|
| `HEALTHCHECK_GRAPHQL_URL` | The URL to check the health of the graphql endpoint | `` |
|
||||||
|
| `BACKEND_HEALTHCHECK_URL` | Optional HTTP readiness URL for startup and periodic backend checks; empty keeps the GraphQL probe | `` |
|
||||||
| `JWT_USER_CLAIM_PATH` | Path to the user claim in the JWT token | `` |
|
| `JWT_USER_CLAIM_PATH` | Path to the user claim in the JWT token | `` |
|
||||||
| `JWT_ROLE_CLAIM_PATH` | Path to the role claim in the JWT token | `` |
|
| `JWT_ROLE_CLAIM_PATH` | Path to the role claim in the JWT token | `` |
|
||||||
| `JWT_VERIFY_SIGNATURE` | Verify the JWT signature before trusting its claims (see [Caching](#caching)) | `false` |
|
| `JWT_VERIFY_SIGNATURE` | Verify the JWT signature before trusting its claims (see [Caching](#caching)) | `false` |
|
||||||
@@ -1127,6 +1128,12 @@ With the `PURGE_METRICS_ON_CRAWL` enabled, the `graphql_proxy_requests_failed`,
|
|||||||
|
|
||||||
If you'd like the `/healthz` endpoint to perform actual check for the connectivity to the graphql endpoint - set the `HEALTHCHECK_GRAPHQL_URL` environment variable to the exact URL of the graphql endpoint. The query executed will be `query { __typename }` and if the response is not `200 OK` - the healthcheck will fail. Remember that the endpoint is a full URL which you'd like to check, so it should include the protocol, host and path - for example `http://localhost:8080/v1/graphql` and it's NOT the same as value of `HOST_GRAPHQL` environment variable which should provide only the host, without path, ending with slash.
|
If you'd like the `/healthz` endpoint to perform actual check for the connectivity to the graphql endpoint - set the `HEALTHCHECK_GRAPHQL_URL` environment variable to the exact URL of the graphql endpoint. The query executed will be `query { __typename }` and if the response is not `200 OK` - the healthcheck will fail. Remember that the endpoint is a full URL which you'd like to check, so it should include the protocol, host and path - for example `http://localhost:8080/v1/graphql` and it's NOT the same as value of `HOST_GRAPHQL` environment variable which should provide only the host, without path, ending with slash.
|
||||||
|
|
||||||
|
`BACKEND_HEALTHCHECK_URL` controls a separate check: the backend health manager's startup and periodic probes. Set it to an unauthenticated readiness endpoint, for example `http://127.0.0.1:8080/ready`. The manager sends `GET` to the supplied path and query, without a GraphQL body or authentication headers. Any 2xx response is healthy; network errors, timeouts, and other status codes are unhealthy. The request timeout remains five seconds, and redirects are not followed.
|
||||||
|
|
||||||
|
When this variable is empty, the manager continues to send an unauthenticated `POST` containing `{"query":"{__typename}"}` to `HOST_GRAPHQL`. It preserves a supplied path, or appends `/v1/graphql` when there is no path beyond `/`. A configured readiness URL must use HTTP or HTTPS with a nonempty host and no userinfo or fragment. Invalid configuration stops startup; it does not fall back to GraphQL. Do not put secrets in the URL: probe targets are logged.
|
||||||
|
|
||||||
|
This setting does not change `HEALTHCHECK_GRAPHQL_URL`, the API `/healthz` check, GraphQL authorization, or headers on forwarded client requests. Keep the GraphQL endpoint protected; only the readiness endpoint must accept these unauthenticated probes.
|
||||||
|
|
||||||
#### Monitoring endpoint
|
#### Monitoring endpoint
|
||||||
|
|
||||||
Example metrics produced by the proxy:
|
Example metrics produced by the proxy:
|
||||||
|
|||||||
+2
-2
@@ -62,7 +62,7 @@ func TestApiBackendHealth_HealthyManager_Returns200(t *testing.T) {
|
|||||||
defer func() { backendHealthManager = orig }()
|
defer func() { backendHealthManager = orig }()
|
||||||
|
|
||||||
// inject a healthy manager directly (bypassing sync.Once)
|
// inject a healthy manager directly (bypassing sync.Once)
|
||||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", libpack_logger.New())
|
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", "", libpack_logger.New())
|
||||||
mgr.isHealthy.Store(true)
|
mgr.isHealthy.Store(true)
|
||||||
backendHealthManager = mgr
|
backendHealthManager = mgr
|
||||||
|
|
||||||
@@ -86,7 +86,7 @@ func TestApiBackendHealth_UnhealthyManager_Returns503(t *testing.T) {
|
|||||||
orig := backendHealthManager
|
orig := backendHealthManager
|
||||||
defer func() { backendHealthManager = orig }()
|
defer func() { backendHealthManager = orig }()
|
||||||
|
|
||||||
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", libpack_logger.New())
|
mgr := NewBackendHealthManager(&fasthttp.Client{}, "http://localhost:8080", "", libpack_logger.New())
|
||||||
mgr.isHealthy.Store(false)
|
mgr.isHealthy.Store(false)
|
||||||
backendHealthManager = mgr
|
backendHealthManager = mgr
|
||||||
|
|
||||||
|
|||||||
+54
-39
@@ -22,6 +22,8 @@ type BackendHealthManager struct {
|
|||||||
logger *libpack_logger.Logger
|
logger *libpack_logger.Logger
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
backendURL string
|
backendURL string
|
||||||
|
healthCheckURL string
|
||||||
|
useHTTPReadiness bool
|
||||||
checkInterval time.Duration
|
checkInterval time.Duration
|
||||||
maxRetries int
|
maxRetries int
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
@@ -32,11 +34,34 @@ type BackendHealthManager struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewBackendHealthManager creates a new backend health manager
|
// NewBackendHealthManager creates a new backend health manager
|
||||||
func NewBackendHealthManager(client *fasthttp.Client, backendURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
func NewBackendHealthManager(client *fasthttp.Client, backendURL, readinessURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
healthCheckURL := readinessURL
|
||||||
|
if healthCheckURL == "" {
|
||||||
|
healthCheckURL = backendGraphQLHealthURL(backendURL)
|
||||||
|
} else {
|
||||||
|
// Path normalization is a client setting in fasthttp. Use a separate
|
||||||
|
// pool so readiness URLs stay exact without changing forwarded traffic.
|
||||||
|
client = &fasthttp.Client{
|
||||||
|
Dial: client.Dial,
|
||||||
|
DialTimeout: client.DialTimeout,
|
||||||
|
TLSConfig: client.TLSConfig,
|
||||||
|
ReadTimeout: client.ReadTimeout,
|
||||||
|
WriteTimeout: client.WriteTimeout,
|
||||||
|
MaxIdleConnDuration: client.MaxIdleConnDuration,
|
||||||
|
MaxConnDuration: client.MaxConnDuration,
|
||||||
|
ReadBufferSize: client.ReadBufferSize,
|
||||||
|
WriteBufferSize: client.WriteBufferSize,
|
||||||
|
MaxResponseBodySize: client.MaxResponseBodySize,
|
||||||
|
NoDefaultUserAgentHeader: client.NoDefaultUserAgentHeader,
|
||||||
|
DisablePathNormalizing: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
return &BackendHealthManager{
|
return &BackendHealthManager{
|
||||||
client: client,
|
client: client,
|
||||||
backendURL: backendURL,
|
backendURL: backendURL,
|
||||||
|
healthCheckURL: healthCheckURL,
|
||||||
|
useHTTPReadiness: readinessURL != "",
|
||||||
checkInterval: 5 * time.Second,
|
checkInterval: 5 * time.Second,
|
||||||
maxRetries: 30, // 30 * 5s = 2.5 minutes max startup wait
|
maxRetries: 30, // 30 * 5s = 2.5 minutes max startup wait
|
||||||
ctx: ctx,
|
ctx: ctx,
|
||||||
@@ -59,6 +84,7 @@ func (bhm *BackendHealthManager) WaitForBackendReady(timeout time.Duration) erro
|
|||||||
Message: "Waiting for GraphQL backend to become ready",
|
Message: "Waiting for GraphQL backend to become ready",
|
||||||
Pairs: map[string]any{
|
Pairs: map[string]any{
|
||||||
"backend_url": bhm.backendURL,
|
"backend_url": bhm.backendURL,
|
||||||
|
"check_url": bhm.healthCheckURL,
|
||||||
"timeout": timeout.String(),
|
"timeout": timeout.String(),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -149,42 +175,14 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
|||||||
defer fasthttp.ReleaseRequest(req)
|
defer fasthttp.ReleaseRequest(req)
|
||||||
defer fasthttp.ReleaseResponse(resp)
|
defer fasthttp.ReleaseResponse(resp)
|
||||||
|
|
||||||
// Determine the health check URL
|
req.SetRequestURI(bhm.healthCheckURL)
|
||||||
// If backendURL is just "http://host:port" or "http://host:port/", append /v1/graphql
|
if bhm.useHTTPReadiness {
|
||||||
// If it has a path like "/v1/graphql", use that path
|
req.Header.SetMethod(http.MethodGet)
|
||||||
healthCheckURL := bhm.backendURL
|
} else {
|
||||||
hasGraphQLPath := false
|
|
||||||
|
|
||||||
if len(bhm.backendURL) > 0 {
|
|
||||||
// Simple check: if URL has a path component beyond just "/"
|
|
||||||
lastSlash := -1
|
|
||||||
protoEnd := 0
|
|
||||||
if idx := strings.Index(bhm.backendURL, "://"); idx >= 0 {
|
|
||||||
protoEnd = idx + 3
|
|
||||||
}
|
|
||||||
for i := protoEnd; i < len(bhm.backendURL); i++ {
|
|
||||||
if bhm.backendURL[i] == '/' {
|
|
||||||
lastSlash = i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Has path if there's a slash after protocol and it's not the last char or followed by more path
|
|
||||||
hasGraphQLPath = lastSlash >= protoEnd && lastSlash < len(bhm.backendURL)-1
|
|
||||||
|
|
||||||
// If no GraphQL path, append /v1/graphql (standard Hasura endpoint)
|
|
||||||
if !hasGraphQLPath {
|
|
||||||
// Remove trailing slash if present
|
|
||||||
baseURL := strings.TrimSuffix(bhm.backendURL, "/")
|
|
||||||
healthCheckURL = baseURL + "/v1/graphql"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Always send GraphQL introspection query for health check
|
|
||||||
healthQuery := `{"query":"{__typename}"}`
|
|
||||||
req.SetRequestURI(healthCheckURL)
|
|
||||||
req.Header.SetMethod(http.MethodPost)
|
req.Header.SetMethod(http.MethodPost)
|
||||||
req.Header.SetContentType("application/json")
|
req.Header.SetContentType("application/json")
|
||||||
req.SetBody([]byte(healthQuery))
|
req.SetBodyString(`{"query":"{__typename}"}`)
|
||||||
|
}
|
||||||
|
|
||||||
// Short timeout for health checks
|
// Short timeout for health checks
|
||||||
err := bhm.client.DoTimeout(req, resp, 5*time.Second)
|
err := bhm.client.DoTimeout(req, resp, 5*time.Second)
|
||||||
@@ -193,7 +191,7 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
|||||||
Message: "Backend health check failed",
|
Message: "Backend health check failed",
|
||||||
Pairs: map[string]any{
|
Pairs: map[string]any{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
"check_url": healthCheckURL,
|
"check_url": bhm.healthCheckURL,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
return false
|
return false
|
||||||
@@ -207,7 +205,7 @@ func (bhm *BackendHealthManager) checkBackendHealth() bool {
|
|||||||
Message: "Backend returned unhealthy status",
|
Message: "Backend returned unhealthy status",
|
||||||
Pairs: map[string]any{
|
Pairs: map[string]any{
|
||||||
"status_code": statusCode,
|
"status_code": statusCode,
|
||||||
"check_url": healthCheckURL,
|
"check_url": bhm.healthCheckURL,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -284,6 +282,9 @@ func (bhm *BackendHealthManager) Shutdown() {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
bhm.cancel()
|
bhm.cancel()
|
||||||
|
if bhm.useHTTPReadiness {
|
||||||
|
bhm.client.CloseIdleConnections()
|
||||||
|
}
|
||||||
if bhm.logger != nil {
|
if bhm.logger != nil {
|
||||||
bhm.logger.Info(&libpack_logger.LogMessage{
|
bhm.logger.Info(&libpack_logger.LogMessage{
|
||||||
Message: "Backend health manager shut down",
|
Message: "Backend health manager shut down",
|
||||||
@@ -298,9 +299,9 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// InitializeBackendHealth initializes the backend health manager
|
// InitializeBackendHealth initializes the backend health manager
|
||||||
func InitializeBackendHealth(client *fasthttp.Client, backendURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
func InitializeBackendHealth(client *fasthttp.Client, backendURL, readinessURL string, logger *libpack_logger.Logger) *BackendHealthManager {
|
||||||
backendHealthOnce.Do(func() {
|
backendHealthOnce.Do(func() {
|
||||||
backendHealthManager = NewBackendHealthManager(client, backendURL, logger)
|
backendHealthManager = NewBackendHealthManager(client, backendURL, readinessURL, logger)
|
||||||
})
|
})
|
||||||
return backendHealthManager
|
return backendHealthManager
|
||||||
}
|
}
|
||||||
@@ -309,3 +310,17 @@ func InitializeBackendHealth(client *fasthttp.Client, backendURL string, logger
|
|||||||
func GetBackendHealthManager() *BackendHealthManager {
|
func GetBackendHealthManager() *BackendHealthManager {
|
||||||
return backendHealthManager
|
return backendHealthManager
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func backendGraphQLHealthURL(backendURL string) string {
|
||||||
|
if backendURL == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
protoEnd := 0
|
||||||
|
if idx := strings.Index(backendURL, "://"); idx >= 0 {
|
||||||
|
protoEnd = idx + 3
|
||||||
|
}
|
||||||
|
if pathStart := strings.IndexByte(backendURL[protoEnd:], '/'); pathStart >= 0 && protoEnd+pathStart < len(backendURL)-1 {
|
||||||
|
return backendURL
|
||||||
|
}
|
||||||
|
return strings.TrimSuffix(backendURL, "/") + "/v1/graphql"
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,234 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
libpack_logger "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"github.com/valyala/fasthttp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackendReadinessProtectedGraphQL(t *testing.T) {
|
||||||
|
var readyStatus atomic.Int32
|
||||||
|
readyStatus.Store(http.StatusOK)
|
||||||
|
var graphqlCalls atomic.Int32
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/v1/graphql":
|
||||||
|
graphqlCalls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
case "/ready":
|
||||||
|
assert.Equal(t, http.MethodGet, r.Method)
|
||||||
|
assert.Empty(t, r.Header.Get("Authorization"))
|
||||||
|
assert.Empty(t, r.Header.Get("X-Husaria-Admin-Secret"))
|
||||||
|
assert.Empty(t, r.Header.Get("X-Hasura-Admin-Secret"))
|
||||||
|
body, err := io.ReadAll(r.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Empty(t, body)
|
||||||
|
w.WriteHeader(int(readyStatus.Load()))
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
client := &fasthttp.Client{}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
logger := libpack_logger.New().SetOutput(io.Discard)
|
||||||
|
legacy := NewBackendHealthManager(client, server.URL, "", logger)
|
||||||
|
t.Cleanup(legacy.Shutdown)
|
||||||
|
require.False(t, legacy.checkBackendHealth(), "GraphQL must reject the unauthenticated probe")
|
||||||
|
|
||||||
|
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", logger)
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
manager.checkInterval = 10 * time.Millisecond
|
||||||
|
manager.StartHealthChecking()
|
||||||
|
require.NoError(t, manager.WaitForBackendReady(time.Second))
|
||||||
|
require.True(t, manager.IsHealthy())
|
||||||
|
|
||||||
|
readyStatus.Store(http.StatusServiceUnavailable)
|
||||||
|
require.Eventually(t, func() bool { return !manager.IsHealthy() }, time.Second, 5*time.Millisecond)
|
||||||
|
readyStatus.Store(http.StatusOK)
|
||||||
|
require.Eventually(t, manager.IsHealthy, time.Second, 5*time.Millisecond)
|
||||||
|
assert.Equal(t, int32(1), graphqlCalls.Load(), "readiness checks must not query protected GraphQL")
|
||||||
|
require.False(t, legacy.checkBackendHealth(), "readiness must not bypass GraphQL authorization")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessExactTarget(t *testing.T) {
|
||||||
|
const target = "/checks//ready%2Fdatabase?probe=one%2Ftwo&probe=three"
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
assert.Equal(t, http.MethodGet, r.Method)
|
||||||
|
assert.Equal(t, target, r.RequestURI)
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
client := &fasthttp.Client{}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
manager := NewBackendHealthManager(client, "http://unused.invalid", server.URL+target, libpack_logger.New().SetOutput(io.Discard))
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
assert.True(t, manager.checkBackendHealth())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessStatus(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
status int
|
||||||
|
healthy bool
|
||||||
|
}{
|
||||||
|
{name: "good OK", status: http.StatusOK, healthy: true},
|
||||||
|
{name: "good no content", status: http.StatusNoContent, healthy: true},
|
||||||
|
{name: "edge final success", status: 299, healthy: true},
|
||||||
|
{name: "bad redirect", status: http.StatusFound},
|
||||||
|
{name: "bad unauthorized", status: http.StatusUnauthorized},
|
||||||
|
{name: "bad unavailable", status: http.StatusServiceUnavailable},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/ready" {
|
||||||
|
t.Error("readiness probe followed a redirect")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Location", "/redirected")
|
||||||
|
w.WriteHeader(tc.status)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
client := &fasthttp.Client{}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
assert.Equal(t, tc.healthy, manager.checkBackendHealth())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessNetworkFailure(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
server.Close()
|
||||||
|
client := &fasthttp.Client{}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
assert.False(t, manager.checkBackendHealth())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessTimeout(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
<-release
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
t.Cleanup(func() { close(release) })
|
||||||
|
client := &fasthttp.Client{ReadTimeout: 20 * time.Millisecond}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
manager := NewBackendHealthManager(client, server.URL, server.URL+"/ready", libpack_logger.New().SetOutput(io.Discard))
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
assert.False(t, manager.checkBackendHealth())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessLegacyGraphQL(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
configuredPath string
|
||||||
|
requestURI string
|
||||||
|
}{
|
||||||
|
{name: "good bare host", requestURI: "/v1/graphql"},
|
||||||
|
{name: "edge root path", configuredPath: "/", requestURI: "/v1/graphql"},
|
||||||
|
{name: "good custom path and query", configuredPath: "/graphql?health=1", requestURI: "/graphql?health=1"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
assert.Equal(t, http.MethodPost, r.Method)
|
||||||
|
assert.Equal(t, tc.requestURI, r.RequestURI)
|
||||||
|
assert.Equal(t, "application/json", r.Header.Get("Content-Type"))
|
||||||
|
assert.Empty(t, r.Header.Get("Authorization"))
|
||||||
|
body, err := io.ReadAll(r.Body)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.JSONEq(t, `{"query":"{__typename}"}`, string(body))
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
client := &fasthttp.Client{}
|
||||||
|
t.Cleanup(client.CloseIdleConnections)
|
||||||
|
manager := NewBackendHealthManager(client, server.URL+tc.configuredPath, "", libpack_logger.New().SetOutput(io.Discard))
|
||||||
|
t.Cleanup(manager.Shutdown)
|
||||||
|
assert.True(t, manager.checkBackendHealth())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readinessURLWithUserinfo(user *url.Userinfo) string {
|
||||||
|
u := url.URL{Scheme: "http", Host: "example.test", Path: "/ready", User: user}
|
||||||
|
return u.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessURLValidation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
valid bool
|
||||||
|
}{
|
||||||
|
{name: "edge unset", valid: true},
|
||||||
|
{name: "good HTTP", url: "http://127.0.0.1:8080/ready", valid: true},
|
||||||
|
{name: "good HTTPS query", url: "https://example.test/check?database=primary", valid: true},
|
||||||
|
{name: "good IPv6", url: "http://[::1]:8080/ready", valid: true},
|
||||||
|
{name: "bad relative", url: "/ready"},
|
||||||
|
{name: "bad scheme", url: "ftp://example.test/ready"},
|
||||||
|
{name: "bad missing host", url: "http:///ready"},
|
||||||
|
{name: "bad port only host", url: "http://:8080/ready"},
|
||||||
|
{name: "bad opaque", url: "http:ready"},
|
||||||
|
{name: "bad userinfo", url: readinessURLWithUserinfo(url.UserPassword("operator", "secret"))},
|
||||||
|
{name: "bad user only", url: readinessURLWithUserinfo(url.User("operator"))},
|
||||||
|
{name: "bad escaped userinfo", url: strings.Replace(readinessURLWithUserinfo(url.UserPassword("operator", "%zz")), "%25zz", "%zz", 1)},
|
||||||
|
{name: "bad path escape", url: "http://example.test/%zz"},
|
||||||
|
{name: "bad port", url: "http://example.test:invalid/ready"},
|
||||||
|
{name: "bad port range", url: "http://example.test:65536/ready"},
|
||||||
|
{name: "bad fragment", url: "http://example.test/ready#ignored"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
err := validateBackendHealthcheckURL(tc.url)
|
||||||
|
if tc.valid {
|
||||||
|
require.NoError(t, err)
|
||||||
|
} else {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.NotContains(t, err.Error(), tc.url)
|
||||||
|
assert.NotContains(t, err.Error(), "operator")
|
||||||
|
assert.NotContains(t, err.Error(), "secret")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendReadinessInvalidConfigStopsStartup(t *testing.T) {
|
||||||
|
const helper = "GMP_TEST_INVALID_READINESS_CONFIG"
|
||||||
|
if os.Getenv(helper) == "1" {
|
||||||
|
parseConfig()
|
||||||
|
t.Fatal("invalid readiness configuration did not stop startup")
|
||||||
|
}
|
||||||
|
command := exec.Command(os.Args[0], "-test.run=^TestBackendReadinessInvalidConfigStopsStartup$")
|
||||||
|
for _, variable := range os.Environ() {
|
||||||
|
if !strings.HasPrefix(variable, "GMP_BACKEND_HEALTHCHECK_URL=") && !strings.HasPrefix(variable, helper+"=") {
|
||||||
|
command.Env = append(command.Env, variable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
command.Env = append(command.Env, helper+"=1", "GMP_BACKEND_HEALTHCHECK_URL="+readinessURLWithUserinfo(url.UserPassword("operator", "secret")))
|
||||||
|
output, err := command.CombinedOutput()
|
||||||
|
var exitError *exec.ExitError
|
||||||
|
require.ErrorAs(t, err, &exitError)
|
||||||
|
assert.Equal(t, 1, exitError.ExitCode())
|
||||||
|
assert.Contains(t, string(output), "Invalid BACKEND_HEALTHCHECK_URL")
|
||||||
|
assert.NotContains(t, string(output), "operator")
|
||||||
|
assert.NotContains(t, string(output), "secret")
|
||||||
|
}
|
||||||
@@ -77,16 +77,8 @@ func (suite *ConnectionResilienceTestSuite) TearDownTest() {
|
|||||||
|
|
||||||
// TestBackendHealthManager tests the backend health monitoring
|
// TestBackendHealthManager tests the backend health monitoring
|
||||||
func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
||||||
suite.Run("initialization", func() {
|
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
|
||||||
suite.NotNil(healthMgr)
|
|
||||||
suite.Equal(cfg.Server.HostGraphQL, healthMgr.backendURL)
|
|
||||||
suite.Equal(5*time.Second, healthMgr.checkInterval)
|
|
||||||
suite.Equal(30, healthMgr.maxRetries)
|
|
||||||
})
|
|
||||||
|
|
||||||
suite.Run("health check success", func() {
|
suite.Run("health check success", func() {
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||||
isHealthy := healthMgr.checkBackendHealth()
|
isHealthy := healthMgr.checkBackendHealth()
|
||||||
suite.True(isHealthy)
|
suite.True(isHealthy)
|
||||||
suite.GreaterOrEqual(suite.mockServerCalled.Load(), int32(1))
|
suite.GreaterOrEqual(suite.mockServerCalled.Load(), int32(1))
|
||||||
@@ -94,13 +86,13 @@ func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
|||||||
|
|
||||||
suite.Run("health check failure", func() {
|
suite.Run("health check failure", func() {
|
||||||
// Use invalid URL to simulate failure
|
// Use invalid URL to simulate failure
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||||
isHealthy := healthMgr.checkBackendHealth()
|
isHealthy := healthMgr.checkBackendHealth()
|
||||||
suite.False(isHealthy)
|
suite.False(isHealthy)
|
||||||
})
|
})
|
||||||
|
|
||||||
suite.Run("startup readiness with healthy backend", func() {
|
suite.Run("startup readiness with healthy backend", func() {
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||||
err := healthMgr.WaitForBackendReady(10 * time.Second)
|
err := healthMgr.WaitForBackendReady(10 * time.Second)
|
||||||
suite.NoError(err)
|
suite.NoError(err)
|
||||||
suite.True(healthMgr.IsHealthy())
|
suite.True(healthMgr.IsHealthy())
|
||||||
@@ -108,7 +100,7 @@ func (suite *ConnectionResilienceTestSuite) TestBackendHealthManager() {
|
|||||||
|
|
||||||
suite.Run("startup readiness timeout", func() {
|
suite.Run("startup readiness timeout", func() {
|
||||||
// Use invalid URL to simulate backend not ready
|
// Use invalid URL to simulate backend not ready
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||||
err := healthMgr.WaitForBackendReady(2 * time.Second)
|
err := healthMgr.WaitForBackendReady(2 * time.Second)
|
||||||
suite.Error(err)
|
suite.Error(err)
|
||||||
suite.Contains(err.Error(), "did not become ready")
|
suite.Contains(err.Error(), "did not become ready")
|
||||||
@@ -177,7 +169,7 @@ func (suite *ConnectionResilienceTestSuite) TestConnectionPoolManager() {
|
|||||||
func (suite *ConnectionResilienceTestSuite) TestIntegratedHealthManagement() {
|
func (suite *ConnectionResilienceTestSuite) TestIntegratedHealthManagement() {
|
||||||
suite.Run("global initialization", func() {
|
suite.Run("global initialization", func() {
|
||||||
// Initialize global instances
|
// Initialize global instances
|
||||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||||
poolMgr := NewConnectionPoolManager(cfg.Client.FastProxyClient)
|
poolMgr := NewConnectionPoolManager(cfg.Client.FastProxyClient)
|
||||||
|
|
||||||
// Set global instances
|
// Set global instances
|
||||||
@@ -194,7 +186,7 @@ func (suite *ConnectionResilienceTestSuite) TestIntegratedHealthManagement() {
|
|||||||
// and may have already fired earlier in the process (e.g. via parseConfig in
|
// and may have already fired earlier in the process (e.g. via parseConfig in
|
||||||
// another test), in which case it returns whatever the global currently is —
|
// another test), in which case it returns whatever the global currently is —
|
||||||
// which TearDownTest above just nilled.
|
// which TearDownTest above just nilled.
|
||||||
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
healthMgr := NewBackendHealthManager(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, "", cfg.Logger)
|
||||||
backendHealthManager = healthMgr
|
backendHealthManager = healthMgr
|
||||||
|
|
||||||
// Start health checking
|
// Start health checking
|
||||||
@@ -272,7 +264,7 @@ func TestConnectionResilienceSuite(t *testing.T) {
|
|||||||
// periodic checking — otherwise the backend could never be marked healthy
|
// periodic checking — otherwise the backend could never be marked healthy
|
||||||
// again even after it recovers.
|
// again even after it recovers.
|
||||||
func (suite *ConnectionResilienceTestSuite) TestHealthChecksStartAfterStartupTimeout() {
|
func (suite *ConnectionResilienceTestSuite) TestHealthChecksStartAfterStartupTimeout() {
|
||||||
bhm := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", cfg.Logger)
|
bhm := NewBackendHealthManager(cfg.Client.FastProxyClient, "http://invalid-url:99999", "", cfg.Logger)
|
||||||
err := bhm.WaitForBackendReady(200 * time.Millisecond)
|
err := bhm.WaitForBackendReady(200 * time.Millisecond)
|
||||||
suite.Error(err, "expected timeout for unreachable backend")
|
suite.Error(err, "expected timeout for unreachable backend")
|
||||||
|
|
||||||
|
|||||||
@@ -187,7 +187,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
|||||||
client := &fasthttp.Client{}
|
client := &fasthttp.Client{}
|
||||||
|
|
||||||
t.Run("updateHealthStatus healthy→unhealthy transition", func(t *testing.T) {
|
t.Run("updateHealthStatus healthy→unhealthy transition", func(t *testing.T) {
|
||||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||||
defer bhm.Shutdown()
|
defer bhm.Shutdown()
|
||||||
|
|
||||||
// Start healthy
|
// Start healthy
|
||||||
@@ -203,7 +203,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("updateHealthStatus unhealthy→healthy resets counter", func(t *testing.T) {
|
t.Run("updateHealthStatus unhealthy→healthy resets counter", func(t *testing.T) {
|
||||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||||
defer bhm.Shutdown()
|
defer bhm.Shutdown()
|
||||||
|
|
||||||
bhm.isHealthy.Store(false)
|
bhm.isHealthy.Store(false)
|
||||||
@@ -219,7 +219,7 @@ func TestCoverageMicro_BackendHealth(t *testing.T) {
|
|||||||
})
|
})
|
||||||
|
|
||||||
t.Run("GetLastHealthCheck round-trip", func(t *testing.T) {
|
t.Run("GetLastHealthCheck round-trip", func(t *testing.T) {
|
||||||
bhm := NewBackendHealthManager(client, "http://localhost:9999", logger)
|
bhm := NewBackendHealthManager(client, "http://localhost:9999", "", logger)
|
||||||
defer bhm.Shutdown()
|
defer bhm.Shutdown()
|
||||||
|
|
||||||
before := time.Now()
|
before := time.Now()
|
||||||
|
|||||||
@@ -246,6 +246,33 @@ func validateJWTClaimPath(path string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateBackendHealthcheckURL(rawURL string) error {
|
||||||
|
if rawURL == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
u, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
// Parse errors can include credentials from the rejected URL.
|
||||||
|
return fmt.Errorf("must be a valid HTTP or HTTPS URL")
|
||||||
|
}
|
||||||
|
if (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.Opaque != "" {
|
||||||
|
return fmt.Errorf("must use HTTP or HTTPS with a nonempty host")
|
||||||
|
}
|
||||||
|
if u.User != nil {
|
||||||
|
return fmt.Errorf("must not contain userinfo or credentials")
|
||||||
|
}
|
||||||
|
if u.Fragment != "" {
|
||||||
|
return fmt.Errorf("must not contain a fragment")
|
||||||
|
}
|
||||||
|
if port := u.Port(); port != "" {
|
||||||
|
n, err := strconv.Atoi(port)
|
||||||
|
if err != nil || n < 1 || n > 65535 {
|
||||||
|
return fmt.Errorf("must use a port between 1 and 65535")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// parseConfig loads and parses the configuration.
|
// parseConfig loads and parses the configuration.
|
||||||
func parseConfig() {
|
func parseConfig() {
|
||||||
libpack_config.PKG_NAME = "graphql_proxy"
|
libpack_config.PKG_NAME = "graphql_proxy"
|
||||||
@@ -259,6 +286,11 @@ func parseConfig() {
|
|||||||
c.Server.BindAddress = getDetailsFromEnv("BIND_ADDRESS", "")
|
c.Server.BindAddress = getDetailsFromEnv("BIND_ADDRESS", "")
|
||||||
c.Server.HostGraphQL = getDetailsFromEnv("HOST_GRAPHQL", "http://localhost/")
|
c.Server.HostGraphQL = getDetailsFromEnv("HOST_GRAPHQL", "http://localhost/")
|
||||||
c.Server.HostGraphQLReadOnly = getDetailsFromEnv("HOST_GRAPHQL_READONLY", "")
|
c.Server.HostGraphQLReadOnly = getDetailsFromEnv("HOST_GRAPHQL_READONLY", "")
|
||||||
|
c.Server.BackendHealthcheckURL = getDetailsFromEnv("BACKEND_HEALTHCHECK_URL", "")
|
||||||
|
if err := validateBackendHealthcheckURL(c.Server.BackendHealthcheckURL); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "CRITICAL ERROR: Invalid BACKEND_HEALTHCHECK_URL: %v\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
// Client configurations
|
// Client configurations
|
||||||
c.Client.JWTUserClaimPath = getDetailsFromEnv("JWT_USER_CLAIM_PATH", "")
|
c.Client.JWTUserClaimPath = getDetailsFromEnv("JWT_USER_CLAIM_PATH", "")
|
||||||
c.Client.JWTRoleClaimPath = getDetailsFromEnv("JWT_ROLE_CLAIM_PATH", "")
|
c.Client.JWTRoleClaimPath = getDetailsFromEnv("JWT_ROLE_CLAIM_PATH", "")
|
||||||
@@ -678,7 +710,7 @@ func parseConfig() {
|
|||||||
|
|
||||||
// Initialize backend health manager
|
// Initialize backend health manager
|
||||||
if cfg.Server.HostGraphQL != "" {
|
if cfg.Server.HostGraphQL != "" {
|
||||||
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Logger)
|
healthMgr := InitializeBackendHealth(cfg.Client.FastProxyClient, cfg.Server.HostGraphQL, cfg.Server.BackendHealthcheckURL, cfg.Logger)
|
||||||
// Start health checking in background
|
// Start health checking in background
|
||||||
healthMgr.StartHealthChecking()
|
healthMgr.StartHealthChecking()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,6 +67,7 @@ type config struct {
|
|||||||
HostGraphQL string
|
HostGraphQL string
|
||||||
HostGraphQLReadOnly string
|
HostGraphQLReadOnly string
|
||||||
HealthcheckGraphQL string
|
HealthcheckGraphQL string
|
||||||
|
BackendHealthcheckURL string
|
||||||
AllowURLs []string // List of allowed URL paths for access control
|
AllowURLs []string // List of allowed URL paths for access control
|
||||||
// BindAddress is the host portion of every listener's bind address
|
// BindAddress is the host portion of every listener's bind address
|
||||||
// (GraphQL proxy, admin API, monitoring). Empty (the default)
|
// (GraphQL proxy, admin API, monitoring). Empty (the default)
|
||||||
|
|||||||
Reference in New Issue
Block a user