Update go.mod and go.sum

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
fixup! Race condition in parseGraphQLQuery result pooling
2026-06-24 04:31:09 +00:00 · 2025-11-19 03:21:29 +00:00 · 2025-11-18 17:27:55 +00:00 · 2025-11-18 17:03:11 +00:00 · 2025-11-18 17:03:06 +00:00 · 2025-11-18 16:28:58 +00:00
13 changed files with 606 additions and 111 deletions
@@ -425,6 +425,8 @@ You can now specify the read-only GraphQL endpoint by setting the `HOST_GRAPHQL_

 You can check out the [example of combined deployment with RW and read-only hasura](static/kubernetes-single-deployment-with-ro.yaml).

+**Important:** When using a read-only Hasura instance connected to a PostgreSQL read replica, you **must** disable event trigger processing on that instance by setting `HASURA_GRAPHQL_EVENTS_FETCH_INTERVAL=0` in the read-only Hasura container environment variables. This prevents the read-only instance from attempting to process event triggers (which require write access to event log tables), avoiding "cannot set transaction read-write mode during recovery" errors.
+
 ### Resilience

 #### Circuit Breaker Pattern
@@ -723,6 +725,8 @@ Following tables are being cleaned:
 - `hdb_catalog.hdb_cron_event_invocation_logs`
 - `hdb_catalog.hdb_scheduled_event_invocation_logs`

+**Important for RO/RW setups:** The `HASURA_EVENT_METADATA_DB` connection string must point to the **read-write primary database** where the `hdb_catalog` schema resides. The cleaner executes DELETE operations which require write permissions. Do not point this to a read-only replica.
+

 ### Security

@@ -0,0 +1,143 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	fiber "github.com/gofiber/fiber/v2"
+	"github.com/graphql-go/graphql/language/ast"
+	"github.com/graphql-go/graphql/language/parser"
+	"github.com/graphql-go/graphql/language/source"
+	libpack_logger "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
+)
+
+// debugParseGraphQLQuery provides detailed logging for mutation routing analysis
+// This is automatically called when LOG_LEVEL=DEBUG to help identify routing issues
+//
+// It logs:
+//   - GraphQL query structure (operations, selections, directives)
+//   - Final routing decision (which endpoint was chosen)
+//   - Automatic detection of mutations routed to wrong endpoints
+//
+// To enable: Set LOG_LEVEL=DEBUG and restart the proxy
+func debugParseGraphQLQuery(c *fiber.Ctx, query string) {
+	if cfg == nil || cfg.Logger == nil {
+		return
+	}
+
+	cfg.Logger.Info(&libpack_logger.LogMessage{
+		Message: "=== DEBUG: Parsing GraphQL Query ===",
+		Pairs: map[string]interface{}{
+			"query_length":  len(query),
+			"query_preview": truncateString(query, 100),
+		},
+	})
+
+	// Parse the query
+	src := source.NewSource(&source.Source{
+		Body: []byte(query),
+		Name: "Debug GraphQL request",
+	})
+
+	p, err := parser.Parse(parser.ParseParams{Source: src})
+	if err != nil {
+		cfg.Logger.Error(&libpack_logger.LogMessage{
+			Message: "DEBUG: Failed to parse query",
+			Pairs:   map[string]interface{}{"error": err.Error()},
+		})
+		return
+	}
+
+	cfg.Logger.Info(&libpack_logger.LogMessage{
+		Message: "DEBUG: Query parsed successfully",
+		Pairs: map[string]interface{}{
+			"definitions_count": len(p.Definitions),
+		},
+	})
+
+	// Analyze each definition
+	for i, d := range p.Definitions {
+		if oper, ok := d.(*ast.OperationDefinition); ok {
+			operationType := strings.ToLower(oper.Operation)
+			operationName := "unnamed"
+			if oper.Name != nil {
+				operationName = oper.Name.Value
+			}
+
+			// Count selections
+			selectionCount := 0
+			if oper.SelectionSet != nil {
+				selectionCount = len(oper.GetSelectionSet().Selections)
+			}
+
+			cfg.Logger.Info(&libpack_logger.LogMessage{
+				Message: fmt.Sprintf("DEBUG: Definition #%d (OperationDefinition)", i),
+				Pairs: map[string]interface{}{
+					"operation_type":  operationType,
+					"operation_name":  operationName,
+					"selection_count": selectionCount,
+					"is_mutation":     operationType == "mutation",
+					"directive_count": len(oper.Directives),
+				},
+			})
+
+			// Log selections for mutations
+			if operationType == "mutation" && oper.SelectionSet != nil {
+				for j, sel := range oper.GetSelectionSet().Selections {
+					if field, ok := sel.(*ast.Field); ok {
+						cfg.Logger.Info(&libpack_logger.LogMessage{
+							Message: fmt.Sprintf("DEBUG: Mutation field #%d", j),
+							Pairs: map[string]interface{}{
+								"field_name": field.Name.Value,
+							},
+						})
+					}
+				}
+			}
+		} else if frag, ok := d.(*ast.FragmentDefinition); ok {
+			cfg.Logger.Info(&libpack_logger.LogMessage{
+				Message: fmt.Sprintf("DEBUG: Definition #%d (FragmentDefinition)", i),
+				Pairs: map[string]interface{}{
+					"fragment_name": frag.Name.Value,
+				},
+			})
+		}
+	}
+
+	// Now run the actual parsing to see the result
+	result := parseGraphQLQuery(c)
+
+	cfg.Logger.Info(&libpack_logger.LogMessage{
+		Message: "DEBUG: Final routing decision",
+		Pairs: map[string]interface{}{
+			"operation_type":  result.operationType,
+			"operation_name":  result.operationName,
+			"active_endpoint": result.activeEndpoint,
+			"should_block":    result.shouldBlock,
+			"should_ignore":   result.shouldIgnore,
+			"write_endpoint":  cfg.Server.HostGraphQL,
+			"read_endpoint":   cfg.Server.HostGraphQLReadOnly,
+			"is_using_write":  result.activeEndpoint == cfg.Server.HostGraphQL,
+		},
+	})
+
+	// Check for potential issues
+	if result.operationType == "mutation" && result.activeEndpoint != cfg.Server.HostGraphQL {
+		cfg.Logger.Error(&libpack_logger.LogMessage{
+			Message: "DEBUG: ⚠️  BUG DETECTED: Mutation routed to wrong endpoint!",
+			Pairs: map[string]interface{}{
+				"expected_endpoint": cfg.Server.HostGraphQL,
+				"actual_endpoint":   result.activeEndpoint,
+			},
+		})
+	}
+
+	if result.operationType == "mutation" && strings.Contains(strings.ToLower(result.activeEndpoint), "read") {
+		cfg.Logger.Error(&libpack_logger.LogMessage{
+			Message: "DEBUG: ⚠️  CRITICAL: Mutation endpoint contains 'read' in URL!",
+			Pairs: map[string]interface{}{
+				"endpoint": result.activeEndpoint,
+			},
+		})
+	}
+}
@@ -15,12 +15,13 @@ const (
 )

 // Use parameterized queries to prevent SQL injection
+// Cast $1 to interval type to allow proper parameterized interval values
 var delQueries = [...]string{
-	"DELETE FROM hdb_catalog.event_invocation_logs WHERE created_at < NOW() - INTERVAL $1",
-	"DELETE FROM hdb_catalog.event_log WHERE created_at < NOW() - INTERVAL $1",
-	"DELETE FROM hdb_catalog.hdb_action_log WHERE created_at < NOW() - INTERVAL $1",
-	"DELETE FROM hdb_catalog.hdb_cron_event_invocation_logs WHERE created_at < NOW() - INTERVAL $1",
-	"DELETE FROM hdb_catalog.hdb_scheduled_event_invocation_logs WHERE created_at < NOW() - INTERVAL $1",
+	"DELETE FROM hdb_catalog.event_invocation_logs WHERE created_at < NOW() - $1::INTERVAL",
+	"DELETE FROM hdb_catalog.event_log WHERE created_at < NOW() - $1::INTERVAL",
+	"DELETE FROM hdb_catalog.hdb_action_log WHERE created_at < NOW() - $1::INTERVAL",
+	"DELETE FROM hdb_catalog.hdb_cron_event_invocation_logs WHERE created_at < NOW() - $1::INTERVAL",
+	"DELETE FROM hdb_catalog.hdb_scheduled_event_invocation_logs WHERE created_at < NOW() - $1::INTERVAL",
 }

 func enableHasuraEventCleaner(ctx context.Context) error {
@@ -340,8 +340,8 @@ func getDelQueries() []string {
 	// This should return the actual delQueries from the main package
 	// For testing purposes, we return expected parameterized queries
 	return []string{
-		"DELETE FROM hdb_catalog.event_log WHERE created_at < NOW() - INTERVAL '$1 days'",
-		"DELETE FROM hdb_catalog.event_invocation_logs WHERE created_at < NOW() - INTERVAL '$1 days'",
+		"DELETE FROM hdb_catalog.event_log WHERE created_at < NOW() - $1::INTERVAL",
+		"DELETE FROM hdb_catalog.event_invocation_logs WHERE created_at < NOW() - $1::INTERVAL",
 	}
 }

@@ -7,28 +7,28 @@ toolchain go1.24.6
 require (
 	github.com/VictoriaMetrics/metrics v1.40.2
 	github.com/alicebob/miniredis/v2 v2.33.0
-	github.com/avast/retry-go/v4 v4.6.1
+	github.com/avast/retry-go/v4 v4.7.0
 	github.com/goccy/go-json v0.10.5
 	github.com/gofiber/fiber/v2 v2.52.9
 	github.com/gofiber/websocket/v2 v2.2.1
-	github.com/gofrs/flock v0.12.1
+	github.com/gofrs/flock v0.13.0
 	github.com/google/uuid v1.6.0
-	github.com/gookit/goutil v0.7.1
+	github.com/gookit/goutil v0.7.2
 	github.com/gorilla/websocket v1.5.3
 	github.com/graphql-go/graphql v0.8.1
 	github.com/jackc/pgx/v5 v5.7.6
 	github.com/lukaszraczylo/ask v0.0.0-20240916204100-6e9ef53a62d9
 	github.com/lukaszraczylo/go-ratecounter v0.1.12
-	github.com/lukaszraczylo/go-simple-graphql v1.2.78
-	github.com/redis/go-redis/v9 v9.14.0
+	github.com/lukaszraczylo/go-simple-graphql v1.2.89
+	github.com/redis/go-redis/v9 v9.16.0
 	github.com/sony/gobreaker v1.0.0
 	github.com/stretchr/testify v1.11.1
-	github.com/valyala/fasthttp v1.66.0
+	github.com/valyala/fasthttp v1.68.0
 	go.opentelemetry.io/otel v1.38.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
 	go.opentelemetry.io/otel/sdk v1.38.0
 	go.opentelemetry.io/otel/trace v1.38.0
-	google.golang.org/grpc v1.75.1
+	google.golang.org/grpc v1.77.0
 )

 require (
@@ -36,17 +36,18 @@ require (
 	github.com/andybalholm/brotli v1.2.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
+	github.com/clipperhouse/stringish v0.1.1 // indirect
+	github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/fasthttp/websocket v1.5.12 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.1 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.19 // indirect
@@ -59,15 +60,15 @@ require (
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
 	go.opentelemetry.io/otel/metric v1.38.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.8.0 // indirect
-	golang.org/x/crypto v0.42.0 // indirect
-	golang.org/x/net v0.44.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.36.0 // indirect
-	golang.org/x/term v0.35.0 // indirect
-	golang.org/x/text v0.29.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250929231259-57b25ae835d4 // indirect
-	google.golang.org/protobuf v1.36.9 // indirect
+	go.opentelemetry.io/proto/otlp v1.9.0 // indirect
+	golang.org/x/crypto v0.44.0 // indirect
+	golang.org/x/net v0.47.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/term v0.37.0 // indirect
+	golang.org/x/text v0.31.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba // indirect
+	google.golang.org/protobuf v1.36.10 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
@@ -6,8 +6,8 @@ github.com/alicebob/miniredis/v2 v2.33.0 h1:uvTF0EDeu9RLnUEG27Db5I68ESoIxTiXbNUi
 github.com/alicebob/miniredis/v2 v2.33.0/go.mod h1:MhP4a3EU7aENRi9aO+tHfTBZicLqQevyi/DJpoj6mi0=
 github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
 github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
-github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk=
-github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA=
+github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio=
+github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -16,8 +16,10 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
 github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
-github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
+github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
+github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
+github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
+github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -38,22 +40,22 @@ github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5
 github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
 github.com/gofiber/websocket/v2 v2.2.1 h1:C9cjxvloojayOp9AovmpQrk8VqvVnT8Oao3+IUygH7w=
 github.com/gofiber/websocket/v2 v2.2.1/go.mod h1:Ao/+nyNnX5u/hIFPuHl28a+NIkrqK7PRimyKaj4JxVU=
-github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E=
-github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0=
+github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
+github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gookit/goutil v0.7.1 h1:AaFJPN9mrdeYBv8HOybri26EHGCC34WJVT7jUStGJsI=
-github.com/gookit/goutil v0.7.1/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
+github.com/gookit/goutil v0.7.2 h1:NSiqWWY+BT0MwIlKDeSVPfQmr9xTkkAqwDjhplobdgo=
+github.com/gookit/goutil v0.7.2/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/graphql-go/graphql v0.8.1 h1:p7/Ou/WpmulocJeEx7wjQy611rtXGQaAcXGqanuMMgc=
 github.com/graphql-go/graphql v0.8.1/go.mod h1:nKiHzRM0qopJEwCITUuIsxk9PlVlwIiiI8pnJEhordQ=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU=
-github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -62,8 +64,8 @@ github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk=
 github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
+github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -72,8 +74,8 @@ github.com/lukaszraczylo/ask v0.0.0-20240916204100-6e9ef53a62d9 h1:pL8B9mjv6RPUf
 github.com/lukaszraczylo/ask v0.0.0-20240916204100-6e9ef53a62d9/go.mod h1:M+UVdyqZs++xtEPrascaVmZdOMhCnxjZ2SgH+xHpR0c=
 github.com/lukaszraczylo/go-ratecounter v0.1.12 h1:VO6hHYGw/Jy9JUizXf/bS0AI2QX1ueWWAWckMFVJ/w4=
 github.com/lukaszraczylo/go-ratecounter v0.1.12/go.mod h1:TqXEOCtFJStk1i0tkipprv1kiDHGon1MVUisjSTBSKM=
-github.com/lukaszraczylo/go-simple-graphql v1.2.78 h1:Ze+vTC1v3QkVB8++EO1gxyA1f/1DbXRgMFrOQDMtSWk=
-github.com/lukaszraczylo/go-simple-graphql v1.2.78/go.mod h1:PxQYblQDZISmYYj8sNfazAWxAOh1rhAtU208y+uPV8s=
+github.com/lukaszraczylo/go-simple-graphql v1.2.89 h1:Xbu1Ny+a0lT2Sr2SaSC8mcHmGQDwGD4TJKk4DDd+PwA=
+github.com/lukaszraczylo/go-simple-graphql v1.2.89/go.mod h1:PxQYblQDZISmYYj8sNfazAWxAOh1rhAtU208y+uPV8s=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -82,8 +84,8 @@ github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byF
 github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE=
-github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
+github.com/redis/go-redis/v9 v9.16.0 h1:OotgqgLSRCmzfqChbQyG1PHC3tLNR89DG4jdOERSEP4=
+github.com/redis/go-redis/v9 v9.16.0/go.mod h1:u410H11HMLoB+TP67dz8rL9s6QW2j76l0//kSOd3370=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/savsgio/gotils v0.0.0-20250924091648-bce9a52d7761 h1:McifyVxygw1d67y6vxUqls2D46J8W9nrki9c8c0eVvE=
@@ -97,8 +99,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.66.0 h1:M87A0Z7EayeyNaV6pfO3tUTUiYO0dZfEJnRGXTVNuyU=
-github.com/valyala/fasthttp v1.66.0/go.mod h1:Y4eC+zwoocmXSVCB1JmhNbYtS7tZPRI2ztPB72EVObs=
+github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok=
+github.com/valyala/fasthttp v1.68.0/go.mod h1:5EXiRfYQAoiO/khu4oU9VISC/eVY6JqmSpPJoHCKsz4=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
@@ -123,33 +125,33 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6
 go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
 go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
 go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8N5CE=
-go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0=
+go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A=
+go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
-golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
-golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
-golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
+golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
-golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ=
-golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
-golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
-golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 h1:8XJ4pajGwOlasW+L13MnEGA8W4115jJySQtVfS2/IBU=
-google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4/go.mod h1:NnuHhy+bxcg30o7FnVAZbXsPHUDQ9qKWAQKCD7VxFtk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250929231259-57b25ae835d4 h1:i8QOKZfYg6AbGVZzUAY3LrNWCKF8O6zFisU9Wl9RER4=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250929231259-57b25ae835d4/go.mod h1:HSkG/KdJWusxU1F6CNrwNDjBMgisKxGnc5dAZfT0mjQ=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
-google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
-google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba h1:B14OtaXuMaCQsl2deSvNkyPKIzq3BjfxQp8d00QyWx4=
+google.golang.org/genproto/googleapis/api v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:G5IanEx8/PgI9w6CFcYQf7jMtHQhZruvfM1i3qOqk5U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba h1:UKgtfRM7Yh93Sya0Fo8ZzhDP4qBckrrxEr2oF5UIVb8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251111163417-95abcf5c77ba/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
+google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -7,6 +7,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+	"unicode"

 	"github.com/goccy/go-json"
 	fiber "github.com/gofiber/fiber/v2"
@@ -37,6 +38,40 @@ var (
 	currentCacheSize  int64 // Use atomic operations for this
 )

+// sanitizeOperationName removes null bytes and other invalid characters from operation names
+// This prevents panics when creating metrics with invalid label values
+func sanitizeOperationName(name string) string {
+	if name == "" || name == "undefined" {
+		return name
+	}
+
+	var buf strings.Builder
+	buf.Grow(len(name))
+
+	for _, r := range name {
+		// Skip null bytes entirely
+		if r == '\x00' {
+			continue
+		}
+		// Replace control characters with underscores
+		if r < 32 || r == 127 {
+			buf.WriteByte('_')
+			continue
+		}
+		// Only allow printable characters
+		if unicode.IsPrint(r) {
+			buf.WriteRune(r)
+		}
+	}
+
+	result := buf.String()
+	// Return "undefined" if we ended up with an empty string after sanitization
+	if result == "" {
+		return "undefined"
+	}
+	return result
+}
+
 func prepareQueriesAndExemptions() {
 	introspectionAllowedQueries = make(map[string]struct{})
 	allowedUrls = make(map[string]struct{})
@@ -298,8 +333,8 @@ func parseGraphQLQuery(c *fiber.Ctx) *parseGraphQLQueryResult {
 				res.operationType = "mutation"
 				if oper.Name != nil {
 					mutationName = oper.Name.Value
-					// Use mutation name immediately
-					res.operationName = mutationName
+					// Use mutation name immediately, sanitized to prevent metric panics
+					res.operationName = sanitizeOperationName(mutationName)
 				}
 				break // Found a mutation, no need to continue first pass
 			}
@@ -316,7 +351,7 @@ func parseGraphQLQuery(c *fiber.Ctx) *parseGraphQLQueryResult {
 				// We already set operation type to mutation in first pass
 				// Only set name if we didn't find a mutation name earlier
 				if res.operationName == "undefined" && oper.Name != nil {
-					res.operationName = oper.Name.Value
+					res.operationName = sanitizeOperationName(oper.Name.Value)
 				}
 			} else {
 				// No mutation found, use the normal logic
@@ -325,18 +360,10 @@ func parseGraphQLQuery(c *fiber.Ctx) *parseGraphQLQueryResult {
 				}

 				if res.operationName == "undefined" && oper.Name != nil {
-					res.operationName = oper.Name.Value
+					res.operationName = sanitizeOperationName(oper.Name.Value)
 				}
 			}

-			// Handle endpoint routing - always use write endpoint for mutations
-			if res.operationType == "mutation" {
-				res.activeEndpoint = cfg.Server.HostGraphQL
-			} else if cfg.Server.HostGraphQLReadOnly != "" {
-				// Use read-only endpoint for non-mutation operations
-				res.activeEndpoint = cfg.Server.HostGraphQLReadOnly
-			}
-
 			// Block mutations in read-only mode
 			if res.operationType == "mutation" && cfg.Server.ReadOnlyMode {
 				if ifNotInTest() {
@@ -359,13 +386,25 @@ func parseGraphQLQuery(c *fiber.Ctx) *parseGraphQLQueryResult {
 		}
 	}

+	// Handle endpoint routing AFTER processing all definitions
+	// This ensures mutations are always routed to the write endpoint
+	if res.operationType == "mutation" {
+		res.activeEndpoint = cfg.Server.HostGraphQL
+	} else if cfg.Server.HostGraphQLReadOnly != "" {
+		// Use read-only endpoint for non-mutation operations
+		res.activeEndpoint = cfg.Server.HostGraphQLReadOnly
+	}
+
 	// Track parsing time
 	if ifNotInTest() && cfg.Monitoring != nil {
 		parseTime := float64(time.Since(startTime).Milliseconds())
 		cfg.Monitoring.IncrementFloat(libpack_monitoring.MetricsGraphQLParsingTime, nil, parseTime)
 	}

-	return res
+	// Create a copy to return, since the original will be returned to the pool
+	// This prevents race conditions where concurrent requests could modify the same result
+	result := *res
+	return &result
 }

 // processDirectives extracts caching directives from the operation
@@ -68,20 +68,82 @@ func ensureDefaultLabels(labels *map[string]string, podName string) {
 	}
 }

-func appendSortedLabels(buf *bytes.Buffer, labels map[string]string) {
-	keys := getSortedKeys(labels)
-	for i, k := range keys {
-		if i > 0 {
-			buf.WriteByte(',')
+// sanitizeLabelValue removes or replaces characters that are invalid in metric labels
+// This includes null bytes, newlines, carriage returns, quotes, and backslashes
+func sanitizeLabelValue(value string) string {
+	if value == "" {
+		return value
+	}
+
+	var buf strings.Builder
+	buf.Grow(len(value))
+
+	for _, r := range value {
+		switch r {
+		case '\x00': // null byte
+			continue // Skip null bytes entirely
+		case '\n', '\r', '\t': // newlines, carriage returns, tabs
+			buf.WriteByte(' ') // Replace with space
+		case '"', '\\': // quotes and backslashes need escaping
+			buf.WriteByte('\\')
+			buf.WriteRune(r)
+		default:
+			// Only allow printable ASCII and common unicode characters
+			if unicode.IsPrint(r) {
+				buf.WriteRune(r)
+			}
+		}
+	}
+
+	return buf.String()
+}
+
+func appendSortedLabels(buf *bytes.Buffer, labels map[string]string) {
+	// Add defer/recover to prevent panics from crashing the application
+	defer func() {
+		if r := recover(); r != nil {
+			// Log the panic but don't crash
+			fmt.Fprintf(os.Stderr, "Recovered from panic in appendSortedLabels: %v\n", r)
+		}
+	}()
+
+	if len(labels) == 0 || buf == nil {
+		return
+	}
+
+	// Create a snapshot to avoid concurrent access issues
+	labelsCopy := make(map[string]string, len(labels))
+	for k, v := range labels {
+		if k == "" {
+			continue // Skip empty keys
+		}
+		// Sanitize the label value to remove null bytes and other invalid characters
+		labelsCopy[k] = sanitizeLabelValue(v)
+	}
+
+	if len(labelsCopy) == 0 {
+		return
+	}
+
+	keys := getSortedKeys(labelsCopy)
+	for i, k := range keys {
+		if v, ok := labelsCopy[k]; ok {
+			if i > 0 {
+				buf.WriteByte(',')
+			}
+			buf.WriteString(k)
+			buf.WriteString(`="`)
+			buf.WriteString(v)
+			buf.WriteByte('"')
 		}
-		buf.WriteString(k)
-		buf.WriteString(`="`)
-		buf.WriteString(labels[k])
-		buf.WriteByte('"')
 	}
 }

 func getSortedKeys(labels map[string]string) []string {
+	if labels == nil {
+		return []string{}
+	}
+
 	labelsKey := labelsToString(labels)

 	// Check if the sorted keys are already cached
@@ -89,7 +151,7 @@ func getSortedKeys(labels map[string]string) []string {
 		return keys.([]string)
 	}

-	// Compute the sorted keys
+	// Compute the sorted keys - create a snapshot to avoid concurrent access issues
 	keys := make([]string, 0, len(labels))
 	for k := range labels {
 		keys = append(keys, k)
@@ -103,18 +165,51 @@ func getSortedKeys(labels map[string]string) []string {
 }

 func labelsToString(labels map[string]string) string {
-	keys := make([]string, 0, len(labels))
-	for k := range labels {
-		keys = append(keys, k)
+	// Add defer/recover to prevent panics from crashing the application
+	defer func() {
+		if r := recover(); r != nil {
+			// Log the panic but don't crash
+			fmt.Fprintf(os.Stderr, "Recovered from panic in labelsToString: %v\n", r)
+		}
+	}()
+
+	if len(labels) == 0 {
+		return ""
 	}
+
+	// Create a snapshot of the map to avoid concurrent access issues
+	keys := make([]string, 0, len(labels))
+	values := make(map[string]string, len(labels))
+
+	for k, v := range labels {
+		if k == "" {
+			continue // Skip empty keys
+		}
+		keys = append(keys, k)
+		values[k] = v
+	}
+
+	if len(keys) == 0 {
+		return ""
+	}
+
 	sort.Strings(keys)

+	// Pre-allocate the builder with estimated capacity to avoid reallocation
 	var sb strings.Builder
+	estimatedSize := 0
 	for _, k := range keys {
-		sb.WriteString(k)
-		sb.WriteByte('=')
-		sb.WriteString(labels[k])
-		sb.WriteByte(';')
+		estimatedSize += len(k) + len(values[k]) + 2 // key + value + '=' + ';'
+	}
+	sb.Grow(estimatedSize)
+
+	for _, k := range keys {
+		if v, ok := values[k]; ok {
+			sb.WriteString(k)
+			sb.WriteByte('=')
+			sb.WriteString(v)
+			sb.WriteByte(';')
+		}
 	}
 	return sb.String()
 }
@@ -164,17 +259,44 @@ func is_special_rune(r rune) bool {
 }

 func compile_metrics_with_labels(name string, labels map[string]string) string {
+	// Add defer/recover to prevent panics from crashing the application
+	defer func() {
+		if r := recover(); r != nil {
+			// Log the panic but don't crash
+			fmt.Fprintf(os.Stderr, "Recovered from panic in compile_metrics_with_labels: %v\n", r)
+		}
+	}()
+
 	var buf bytes.Buffer

 	buf.WriteString(name)

-	keys := getSortedKeys(labels)
+	if len(labels) == 0 {
+		return buf.String()
+	}
+
+	// Create a snapshot to avoid concurrent access issues
+	labelsCopy := make(map[string]string, len(labels))
+	for k, v := range labels {
+		if k == "" {
+			continue // Skip empty keys
+		}
+		labelsCopy[k] = v
+	}
+
+	if len(labelsCopy) == 0 {
+		return buf.String()
+	}
+
+	keys := getSortedKeys(labelsCopy)

 	for _, k := range keys {
-		buf.WriteByte('_')
-		buf.WriteString(k)
-		buf.WriteByte('_')
-		buf.WriteString(labels[k])
+		if v, ok := labelsCopy[k]; ok {
+			buf.WriteByte('_')
+			buf.WriteString(k)
+			buf.WriteByte('_')
+			buf.WriteString(v)
+		}
 	}

 	return buf.String()
@@ -82,6 +82,36 @@ func (suite *Tests) Test_proxyTheRequest() {
 			wantErr:      false,
 			wantEndpoint: "https://telegram-bot.app/",
 		},
+		{
+			name:         "Test mutation with multiple operations (bug fix regression test)",
+			body:         `{"query":"mutation getOrCreateUser { insert_tg_users_one(object: {id: 123}) { id } } query otherQuery { users { id } }"}`,
+			host:         "https://telegram-bot.app/",
+			hostRO:       "https://google.com/",
+			path:         "/v1/graphql",
+			headers:      supplied_headers,
+			wantErr:      false,
+			wantEndpoint: "https://telegram-bot.app/",
+		},
+		{
+			name:         "Test mutation followed by fragment (bug fix regression test)",
+			body:         `{"query":"mutation insertUser { insert_users_one(object: {name: \"test\"}) { ...userFields } } fragment userFields on users { id name }"}`,
+			host:         "https://telegram-bot.app/",
+			hostRO:       "https://google.com/",
+			path:         "/v1/graphql",
+			headers:      supplied_headers,
+			wantErr:      false,
+			wantEndpoint: "https://telegram-bot.app/",
+		},
+		{
+			name:         "Test complex mutation document (main-bot style)",
+			body:         `{"query":"mutation getOrCreateUser($user_id: bigint!, $group_id: bigint!) { insert_tg_users_one(object: {id: $user_id}, on_conflict: {constraint: tg_users_pkey, update_columns: last_seen}) { id } insert_tg_groups_one(object: {id: $group_id}, on_conflict: {constraint: tg_groups_pkey, update_columns: last_seen}) { id } }"}`,
+			host:         "https://telegram-bot.app/",
+			hostRO:       "https://google.com/",
+			path:         "/v1/graphql",
+			headers:      supplied_headers,
+			wantErr:      false,
+			wantEndpoint: "https://telegram-bot.app/",
+		},
 	}

 	for _, tt := range tests {
@@ -272,6 +272,17 @@ func processGraphQLRequest(c *fiber.Ctx) error {

 	// Parse the GraphQL query
 	parsedResult := parseGraphQLQuery(c)
+
+	// Debug logging for mutation routing analysis (enabled when LOG_LEVEL=DEBUG)
+	if cfg.LogLevel == "DEBUG" {
+		var m map[string]interface{}
+		if err := json.Unmarshal(c.Body(), &m); err == nil {
+			if query, ok := m["query"].(string); ok {
+				debugParseGraphQLQuery(c, query)
+			}
+		}
+	}
+
 	if parsedResult.shouldBlock {
 		return c.Status(fiber.StatusForbidden).SendString("Request blocked")
 	}
@@ -97,6 +97,9 @@ spec:
              value: "error"
            - name: HASURA_GRAPHQL_SERVER_PORT
              value: "8088"
+            # Disable event trigger processing on read-only instance
+            - name: HASURA_GRAPHQL_EVENTS_FETCH_INTERVAL
+              value: "0"

        - name: graphql-proxy
          image: ghcr.io/lukaszraczylo/graphql-monitoring-proxy:latest
@@ -8,6 +8,7 @@ import (
 	"sync/atomic"
 	"time"

+	"github.com/goccy/go-json"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/websocket/v2"
 	gorillaws "github.com/gorilla/websocket"
@@ -88,13 +89,39 @@ func (wsp *WebSocketProxy) HandleWebSocket(c *fiber.Ctx) error {
 		return fiber.NewError(fiber.StatusUpgradeRequired, "WebSocket upgrade required")
 	}

+	// Capture headers from the upgrade request to forward to backend
+	headers := make(http.Header)
+	var subprotocols []string
+
+	for key, value := range c.Request().Header.All() {
+		keyStr := string(key)
+		// Capture subprotocol separately
+		if keyStr == "Sec-Websocket-Protocol" || keyStr == "Sec-WebSocket-Protocol" {
+			subprotocols = append(subprotocols, string(value))
+		}
+		// Forward important headers including WebSocket subprotocol
+		// Skip only connection-establishment headers that will be regenerated
+		if keyStr != "Connection" && keyStr != "Upgrade" &&
+			keyStr != "Sec-Websocket-Key" && keyStr != "Sec-Websocket-Version" &&
+			keyStr != "Sec-Websocket-Extensions" {
+			headers.Add(keyStr, string(value))
+		}
+	}
+
+	// Configure WebSocket with subprotocol support
+	config := websocket.Config{
+		Subprotocols: subprotocols,
+	}
+
 	return websocket.New(func(clientConn *websocket.Conn) {
-		wsp.handleConnection(c.Context(), clientConn)
-	})(c)
+		// Use background context for long-lived WebSocket connections
+		// The original request context expires after the upgrade
+		wsp.handleConnection(context.Background(), clientConn, headers)
+	}, config)(c)
 }

 // handleConnection manages a single WebSocket connection
-func (wsp *WebSocketProxy) handleConnection(ctx context.Context, clientConn *websocket.Conn) {
+func (wsp *WebSocketProxy) handleConnection(ctx context.Context, clientConn *websocket.Conn, headers http.Header) {
 	connectionID := fmt.Sprintf("%p", clientConn)
 	startTime := time.Now()

@@ -115,8 +142,29 @@ func (wsp *WebSocketProxy) handleConnection(ctx context.Context, clientConn *web
 	// Set message size limit
 	clientConn.SetReadLimit(wsp.maxMessageSize)

-	// Connect to backend WebSocket
-	backendConn, err := wsp.dialBackend(ctx)
+	// Read first message to extract authentication from connection_init payload
+	// This bridges the gap between clients that send auth in payload vs Hasura expecting it in HTTP headers
+	messageType, message, err := clientConn.ReadMessage()
+	if err != nil {
+		wsp.errors.Add(1)
+		if wsp.logger != nil {
+			wsp.logger.Error(&libpack_logger.LogMessage{
+				Message: "Failed to read first message from client",
+				Pairs: map[string]interface{}{
+					"connection_id": connectionID,
+					"error":         err.Error(),
+				},
+			})
+		}
+		clientConn.Close()
+		return
+	}
+
+	// Try to extract headers from connection_init payload (for GraphQL WebSocket protocols)
+	enrichedHeaders := wsp.extractAuthFromPayload(message, headers)
+
+	// Connect to backend WebSocket with enriched headers
+	backendConn, err := wsp.dialBackend(ctx, enrichedHeaders)
 	if err != nil {
 		wsp.errors.Add(1)
 		if wsp.logger != nil {
@@ -133,6 +181,32 @@ func (wsp *WebSocketProxy) handleConnection(ctx context.Context, clientConn *web
 	}
 	defer backendConn.Close()

+	// Forward the first message (connection_init) to backend
+	if err := backendConn.WriteMessage(messageType, message); err != nil {
+		wsp.errors.Add(1)
+		if wsp.logger != nil {
+			wsp.logger.Error(&libpack_logger.LogMessage{
+				Message: "Failed to forward connection_init to backend",
+				Pairs: map[string]interface{}{
+					"connection_id": connectionID,
+					"error":         err.Error(),
+				},
+			})
+		}
+		return
+	}
+
+	if wsp.logger != nil {
+		wsp.logger.Debug(&libpack_logger.LogMessage{
+			Message: "Backend WebSocket connection established",
+			Pairs: map[string]interface{}{
+				"connection_id":     connectionID,
+				"subprotocol":       backendConn.Subprotocol(),
+				"has_authorization": headers.Get("Authorization") != "",
+			},
+		})
+	}
+
 	// Set up bidirectional proxying
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -299,8 +373,60 @@ func (wsp *WebSocketProxy) proxyBackendToClient(ctx context.Context, backend *go
 	}
 }

+// extractAuthFromPayload extracts authentication headers from GraphQL WebSocket connection_init payload
+// This bridges the gap between clients sending auth in payload and Hasura expecting it in HTTP headers
+func (wsp *WebSocketProxy) extractAuthFromPayload(message []byte, originalHeaders http.Header) http.Header {
+	// Create a copy of original headers
+	enrichedHeaders := make(http.Header)
+	for k, v := range originalHeaders {
+		enrichedHeaders[k] = v
+	}
+
+	// Try to parse as JSON to extract headers from payload
+	var msg map[string]interface{}
+	if err := json.Unmarshal(message, &msg); err != nil {
+		// Not JSON or parse error, return original headers
+		return enrichedHeaders
+	}
+
+	// Check if this is a connection_init message
+	msgType, ok := msg["type"].(string)
+	if !ok || (msgType != "connection_init" && msgType != "start") {
+		// Not a connection_init, return original headers
+		return enrichedHeaders
+	}
+
+	// Extract payload
+	payload, ok := msg["payload"].(map[string]interface{})
+	if !ok {
+		return enrichedHeaders
+	}
+
+	// Try to extract headers from payload.headers (graphql-ws format)
+	if payloadHeaders, ok := payload["headers"].(map[string]interface{}); ok {
+		for key, value := range payloadHeaders {
+			if strValue, ok := value.(string); ok {
+				enrichedHeaders.Set(key, strValue)
+			}
+		}
+	}
+
+	// Also check top-level payload keys that look like headers (Apollo format)
+	for key, value := range payload {
+		if strValue, ok := value.(string); ok {
+			// Common auth headers
+			if key == "Authorization" || key == "authorization" ||
+				key == "x-hasura-role" || key == "x-hasura-admin-secret" {
+				enrichedHeaders.Set(key, strValue)
+			}
+		}
+	}
+
+	return enrichedHeaders
+}
+
 // dialBackend establishes a WebSocket connection to the backend
-func (wsp *WebSocketProxy) dialBackend(ctx context.Context) (*gorillaws.Conn, error) {
+func (wsp *WebSocketProxy) dialBackend(ctx context.Context, headers http.Header) (*gorillaws.Conn, error) {
 	// Convert http:// to ws:// or https:// to wss://
 	wsURL := wsp.backendURL
 	if len(wsURL) > 7 && wsURL[:7] == "http://" {
@@ -309,13 +435,24 @@ func (wsp *WebSocketProxy) dialBackend(ctx context.Context) (*gorillaws.Conn, er
 		wsURL = "wss://" + wsURL[8:]
 	}

+	// Append GraphQL WebSocket path
+	wsURL = wsURL + "/v1/graphql"
+
+	// Extract subprotocols from headers (e.g., graphql-ws, graphql-transport-ws)
+	var subprotocols []string
+	if proto := headers.Get("Sec-WebSocket-Protocol"); proto != "" {
+		subprotocols = []string{proto}
+		// Remove from headers since it will be set via Subprotocols field
+		headers.Del("Sec-WebSocket-Protocol")
+	}
+
 	// Use gorilla websocket dialer
 	dialer := gorillaws.Dialer{
 		HandshakeTimeout: 10 * time.Second,
+		Subprotocols:     subprotocols,
 	}

-	// Dial the backend with proper headers
-	headers := http.Header{}
+	// Dial the backend with forwarded headers
 	conn, _, err := dialer.DialContext(ctx, wsURL, headers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to dial backend WebSocket: %w", err)
@@ -2,6 +2,7 @@ package main

 import (
 	"context"
+	"net/http"
 	"testing"
 	"time"

@@ -167,7 +168,8 @@ func TestWebSocketProxy_DialBackend_URLConversion(t *testing.T) {
 			// We can't fully test dialBackend without a real WebSocket server,
 			// but we can verify the URL conversion logic
 			ctx := context.Background()
-			_, err := wsp.dialBackend(ctx)
+			headers := http.Header{}
+			_, err := wsp.dialBackend(ctx, headers)

 			// We expect an error since there's no server, but we verify the conversion happened
 			assert.Error(t, err) // Should fail to connect to non-existent server
Author	SHA1	Message	Date
lukaszraczylo	ee5618c699	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-19 03:21:29 +00:00
lukaszraczylo	94c097bc6c	fixup! Race condition in parseGraphQLQuery result pooling	2025-11-18 17:27:55 +00:00
lukaszraczylo	4e84cd7461	Race condition in parseGraphQLQuery result pooling Under high concurrency, the sync.Pool pattern was creating a race condition where the same result pointer was being reused by multiple concurrent requests. The bug: - parseGraphQLQuery() returns a pointer to 'res' from the pool - The defer statement returns 'res' back to the pool on function exit - While the caller is still using the returned pointer, another concurrent request could get the SAME pointer from the pool and modify it This caused mutations to randomly get the wrong activeEndpoint value: - Request A: mutation parsed → activeEndpoint set to :8080 (write) - Request A: returns pointer to result - Request A: defer runs → result returned to pool - Request B: gets SAME pointer from pool - Request B: query parsed → activeEndpoint overwritten to :8088 (read-only) - Request A: still holding pointer, now sees :8088 instead of :8080! - Result: mutation routed to read-only endpoint → database write failure The fix: Create a copy of the result before returning, so the pooled object can be safely reused without affecting the returned value.	2025-11-18 17:03:11 +00:00
lukaszraczylo	e37a8beaa7	Fix: Move endpoint routing outside loop to prevent mutation misrouting BUG FIX: The endpoint routing logic was inside the loop that processes all GraphQL definitions. This caused mutations to be incorrectly routed to read-only endpoints when followed by other definitions (queries, fragments, etc). The bug manifested as: mutations → read-only Hasura → read-only pooler → PostgreSQL replica → "cannot set transaction read-write mode during recovery" Changes: - Move endpoint routing logic AFTER the definition processing loop - Ensures mutations are ALWAYS routed to write endpoint regardless of subsequent definitions in the document - Add 3 comprehensive regression tests covering: 1. Mutation with multiple operations 2. Mutation followed by fragment 3. Complex main-bot style mutation document Tests: All pass including new regression tests Impact: Fixes database write failures in main-bot and other services	2025-11-18 17:03:06 +00:00
lukaszraczylo	9dd8c11363	CRITICAL: Routing fix for mutations in case of the R/W replicas	2025-11-18 16:28:58 +00:00
lukaszraczylo	9fbee0d9a1	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-18 03:21:43 +00:00
lukaszraczylo	7df651c17a	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-12 03:21:41 +00:00
lukaszraczylo	7ada94e4fa	Fix nil pointers + improve the cleanup.	2025-11-11 10:43:07 +00:00
lukaszraczylo	c510c29a8f	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-11 03:22:29 +00:00
lukaszraczylo	370602858a	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-09 03:21:44 +00:00
lukaszraczylo	6261be6e53	fixup! fixup! fixup! fixup! fixup! fixup! fixup! fixup! Update go.mod and go.sum	2025-11-06 16:55:12 +00:00
lukaszraczylo	5ae4ea1e25	fixup! fixup! fixup! fixup! fixup! fixup! fixup! Update go.mod and go.sum	2025-11-05 22:55:03 +00:00
lukaszraczylo	fd30dc0890	fixup! fixup! fixup! fixup! fixup! fixup! Update go.mod and go.sum	2025-11-05 21:56:29 +00:00
lukaszraczylo	2966661054	fixup! fixup! fixup! fixup! fixup! Update go.mod and go.sum	2025-11-05 21:47:40 +00:00
lukaszraczylo	0f23f10e2f	fixup! fixup! fixup! fixup! Update go.mod and go.sum	2025-11-05 21:34:21 +00:00
lukaszraczylo	ce39dc1bee	fixup! fixup! fixup! Update go.mod and go.sum	2025-11-05 20:58:41 +00:00
lukaszraczylo	f864e8edcf	fixup! fixup! Update go.mod and go.sum	2025-11-05 20:28:31 +00:00
lukaszraczylo	e36cdf099e	fixup! Update go.mod and go.sum	2025-11-05 10:27:18 +00:00
lukaszraczylo	e2c3d03661	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-11-04 03:20:22 +00:00
lukaszraczylo	9de8b7bcaa	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-30 03:20:35 +00:00
lukaszraczylo	163fc5ac42	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-27 03:26:43 +00:00
lukaszraczylo	758412e54e	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-24 03:15:12 +00:00
lukaszraczylo	2f3909c5b0	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-23 03:16:57 +00:00
lukaszraczylo	737e349b66	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-21 03:20:10 +00:00
lukaszraczylo	55c6843c8c	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-19 03:23:35 +00:00
lukaszraczylo	d6534ed519	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-15 03:16:59 +00:00
lukaszraczylo	d471948e19	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-10 03:14:45 +00:00
lukaszraczylo	a8959b6afa	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-09 03:14:30 +00:00
lukaszraczylo	bb4979587b	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-08 03:12:08 +00:00
lukaszraczylo	58f511103d	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-07 03:12:36 +00:00
lukaszraczylo	9b74334a15	Update go.mod and go.sum Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>	2025-10-03 03:11:19 +00:00