fix(coalesce): copy the primary response body for waiters

Waiters read the primary response buffer while fasthttp could reuse it for another request. A waiter could then send the wrong body. The primary now gives waiters their own copy.

.gitignore

@@ -5,3 +5,6 @@ dist/
 coverage.out
 CLAUDE.md
 graphql-monitoring-proxy
+
+# Lean proofs: local only, never committed
+theorems/

coalesced_snapshot_test.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/gofiber/fiber/v3"
+	"github.com/stretchr/testify/assert"
+	"github.com/valyala/fasthttp"
+)
+
+// Pins theorems/Theorems/CoalesceAlias.lean: waiters never alias the primary's response buffer.
+func TestSnapshotCoalescedResponse_OwnsBody(t *testing.T) {
+	app := fiber.New()
+	fctx := &fasthttp.RequestCtx{}
+	c := app.AcquireCtx(fctx)
+	defer app.ReleaseCtx(c)
+
+	c.Response().SetStatusCode(201)
+	c.Response().SetBody([]byte(`{"user":"alice"}`))
+
+	snap := snapshotCoalescedResponse(c)
+
+	c.Response().SetBody([]byte(`{"user":"bobby"}`)) // buffer reused by a later request
+
+	assert.Equal(t, 201, snap.StatusCode)
+	assert.Equal(t, `{"user":"alice"}`, string(snap.Body))
+}

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/VictoriaMetrics/metrics v1.44.1
 	github.com/alicebob/miniredis/v2 v2.40.0
 	github.com/avast/retry-go/v5 v5.0.0
-	github.com/goccy/go-json v0.11.2
-	github.com/gofiber/contrib/v3/websocket v1.2.9
+	github.com/goccy/go-json v0.12.0
+	github.com/gofiber/contrib/v3/websocket v1.2.10
 	github.com/gofiber/fiber/v3 v3.6.1
 	github.com/gofrs/flock v0.13.1
 	github.com/golang-jwt/jwt/v5 v5.3.1
@@ -18,7 +18,7 @@ require (
 	github.com/gookit/goutil v0.8.6
 	github.com/gorilla/websocket v1.5.3
 	github.com/graphql-go/graphql v0.8.1
-	github.com/jackc/pgx/v5 v5.11.0
+	github.com/jackc/pgx/v5 v5.12.0
 	github.com/lukaszraczylo/ask v0.0.0-20240916204100-6e9ef53a62d9
 	github.com/lukaszraczylo/go-ratecounter v0.1.13
 	github.com/lukaszraczylo/go-simple-graphql v1.2.89
@@ -43,7 +43,7 @@ require (
 	github.com/fasthttp/websocket v1.5.12 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/gofiber/schema v1.9.1 // indirect
+	github.com/gofiber/schema v1.9.2 // indirect
 	github.com/gofiber/utils/v2 v2.6.3 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.33.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect

go.sum

@@ -28,16 +28,16 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8=
 github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
-github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
+github.com/goccy/go-json v0.12.0 h1:pD+oWouAiz4yruwXIt/iT+lOMeyhr1aTtxzaMP/RgVI=
+github.com/goccy/go-json v0.12.0/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
 github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms=
 github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE=
-github.com/gofiber/contrib/v3/websocket v1.2.9 h1:DgZgxWK5Yua3g5+I4TmoA0RRx6MILIaqooSGKMgjWZE=
-github.com/gofiber/contrib/v3/websocket v1.2.9/go.mod h1:+unMmdAcLfIywqYDaH8eTynn503n7Lf4Cr4kXM5ghO0=
+github.com/gofiber/contrib/v3/websocket v1.2.10 h1:tUyRcHSH9GUsh6WO03Xn+Kjvt3850W6raF6WPngdJLE=
+github.com/gofiber/contrib/v3/websocket v1.2.10/go.mod h1:O6MOrNHCVdCrrstdLrYmRIL0eJy2N87Wa/y2dszuxvQ=
 github.com/gofiber/fiber/v3 v3.6.1 h1:wYjbgqpJm6VErIVLSGzGpX7lLZyTNgElsj8nQ1i+iFo=
 github.com/gofiber/fiber/v3 v3.6.1/go.mod h1:+nxjBal3gw0EPf3gqK5EmtxaxTL01+zbSTagFletlDI=
-github.com/gofiber/schema v1.9.1 h1:QjicFdTDMlkzCOoZAlNWqdkR2YHhgH8/Q/MpcbHurjs=
-github.com/gofiber/schema v1.9.1/go.mod h1:219nYAgzIEBWl/aHCyhQtFoXBvfLjsC6+FI3DUN1+Ck=
+github.com/gofiber/schema v1.9.2 h1:Gzw2PF1K5DpWgG2L51/9t9dkmxw4kqK6auKMvgD9mIU=
+github.com/gofiber/schema v1.9.2/go.mod h1:219nYAgzIEBWl/aHCyhQtFoXBvfLjsC6+FI3DUN1+Ck=
 github.com/gofiber/utils/v2 v2.6.3 h1:VlnOB89rFnonlXoYOkESOkXtIkAw4CBBwpBHu+/a/YQ=
 github.com/gofiber/utils/v2 v2.6.3/go.mod h1:Mq3OXwyBPxORb3jPafy6Swki4/TJJW9TDPHeerXkXpg=
 github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo=
@@ -62,8 +62,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
-github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/pgx/v5 v5.12.0 h1:hWwax8YiGT0YiN50U1oIDn5kgR7KC2tG3rmavwpPkxY=
+github.com/jackc/pgx/v5 v5.12.0/go.mod h1:u2Fi6no03HCSBYE+ZVdLxbwdu3xyi2vU+K/YNJioSds=
 github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao=
 github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=

proxy.go

@@ -740,11 +740,7 @@ func performProxyRequest(c fiber.Ctx, proxyURL string) error {
 				}, proxyErr
 			}
 
-			return &CoalescedResponse{
-				Body:       c.Response().Body(),
-				StatusCode: c.Response().StatusCode(),
-				// Headers intentionally left nil; not populated or read anywhere.
-			}, nil
+			return snapshotCoalescedResponse(c), nil
 		})
 
 		// Check for error from rc.Do (though it typically returns nil)
@@ -773,6 +769,17 @@ func performProxyRequest(c fiber.Ctx, proxyURL string) error {
 	return performProxyRequestCore(c, proxyURL, cacheKey)
 }
 
+// snapshotCoalescedResponse captures the primary's response for waiters. It
+// copies the body because fasthttp reuses the buffer once the primary handler
+// returns, while waiters can still be reading.
+func snapshotCoalescedResponse(c fiber.Ctx) *CoalescedResponse {
+	return &CoalescedResponse{
+		Body:       bytes.Clone(c.Response().Body()),
+		StatusCode: c.Response().StatusCode(),
+		// Headers intentionally left nil; not populated or read anywhere.
+	}
+}
+
 // performProxyRequestCore executes the proxy request with retries and circuit breaker
 // This is the core implementation used by both direct calls and coalesced requests
 func performProxyRequestCore(c fiber.Ctx, proxyURL string, cacheKey string) error {

retry_budget.go

@@ -148,31 +148,39 @@ func (rb *RetryBudget) refill() {
 	now := time.Now().UnixNano()
 	last := rb.lastRefill.Load()
 
-	// Calculate elapsed time in seconds
-	elapsed := float64(now-last) / float64(time.Second)
+	tokensToAdd, newLast := refillStep(tokensPerSecond, last, now)
+	if tokensToAdd <= 0 {
+		return
+	}
+	if rb.lastRefill.CompareAndSwap(last, newLast) {
+		for {
+			current := rb.currentTokens.Load()
+			newValue := current + tokensToAdd
+			if newValue > maxTokens {
+				newValue = maxTokens
+			}
 
-	// Calculate tokens to add
-	tokensToAdd := int64(elapsed * tokensPerSecond)
-
-	if tokensToAdd > 0 {
-		// Update last refill time
-		if rb.lastRefill.CompareAndSwap(last, now) {
-			// Add tokens, capped at maxTokens
-			for {
-				current := rb.currentTokens.Load()
-				newValue := current + tokensToAdd
-				if newValue > maxTokens {
-					newValue = maxTokens
-				}
-
-				if rb.currentTokens.CompareAndSwap(current, newValue) {
-					break
-				}
+			if rb.currentTokens.CompareAndSwap(current, newValue) {
+				break
 			}
 		}
 	}
 }
 
+// refillStep returns the whole tokens earned between last and now (ns) and the
+// clock to store. The clock advances only by the time those tokens pay for, so
+// the fractional remainder carries into the next tick.
+func refillStep(tokensPerSecond float64, last, now int64) (int64, int64) {
+	if tokensPerSecond <= 0 || now <= last {
+		return 0, last
+	}
+	tokens := int64(float64(now-last) / float64(time.Second) * tokensPerSecond)
+	if tokens <= 0 {
+		return 0, last
+	}
+	return tokens, last + int64(float64(tokens)/tokensPerSecond*float64(time.Second))
+}
+
 // GetStats returns current statistics
 func (rb *RetryBudget) GetStats() map[string]any {
 	totalAttempts := rb.totalAttempts.Load()

retry_budget_refill_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Pins theorems/Theorems/RetryRefill.lean: total_added(15 tps, 100ms ticks, 10s) = 150.
+func TestRefillStep_KeepsFractionalTokens(t *testing.T) {
+	tests := []struct {
+		name string
+		tps  float64
+		want int64
+	}{
+		{"15 tps", 15, 150},
+		{"25 tps", 25, 250},
+		{"5 tps (sub-token per tick)", 5, 50},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tick := 100 * time.Millisecond
+			var last, total int64
+			now := int64(tick)
+			for i := 0; i < 100; i++ {
+				added, newLast := refillStep(tt.tps, last, now)
+				total += added
+				last = newLast
+				now += int64(tick)
+			}
+			assert.Equal(t, tt.want, total)
+		})
+	}
+}
+
+func TestRefillStep_NoElapsedNoChange(t *testing.T) {
+	added, last := refillStep(10, 1000, 1000)
+	assert.Zero(t, added)
+	assert.Equal(t, int64(1000), last)
+}
+
+func TestRefillStep_ZeroRateNeverAdds(t *testing.T) {
+	added, last := refillStep(0, 0, int64(time.Hour))
+	assert.Zero(t, added)
+	assert.Zero(t, last)
+}

rps_tracker.go

@@ -57,17 +57,23 @@ func (r *RPSTracker) Shutdown() {
 
 // sample calculates RPS since last sample
 func (r *RPSTracker) sample() {
-	now := time.Now()
-	nowNano := now.UnixNano()
+	r.sampleCount()
+}
+
+// sampleCount performs one sample and returns the requests it consumed.
+func (r *RPSTracker) sampleCount() int64 {
+	nowNano := time.Now().UnixNano()
 
-	currentCount := r.lastCount.Load()
 	lastSampleNano := r.lastSampleTime.Load()
 
 	if lastSampleNano == 0 {
 		r.lastSampleTime.Store(nowNano)
-		return
+		return 0
 	}
 
+	// Swap, not Load+Store(0): requests recorded between the two would be lost.
+	currentCount := r.lastCount.Swap(0)
+
 	elapsed := float64(nowNano-lastSampleNano) / float64(time.Second)
 	if elapsed > 0 {
 		rps := float64(currentCount) / elapsed
@@ -75,9 +81,8 @@ func (r *RPSTracker) sample() {
 		r.currentRPS.Store(uint64(rps * 100))
 	}
 
-	// Reset for next sample
-	r.lastCount.Store(0)
 	r.lastSampleTime.Store(nowNano)
+	return currentCount
 }
 
 // GetCurrentRPS returns the current requests per second

rps_tracker_sample_test.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Pins theorems/Theorems/RpsSample.lean: no recorded request is dropped by sample().
+func TestRPSTracker_SampleKeepsConcurrentRequests(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	tr := NewRPSTracker(ctx)
+	tr.Shutdown() // stop the background sampler; sample() is driven by the test
+
+	const writers, perWriter = 8, 20000
+	var wg sync.WaitGroup
+	for i := 0; i < writers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < perWriter; j++ {
+				tr.RecordRequest()
+			}
+		}()
+	}
+	var sampled int64
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+loop:
+	for {
+		select {
+		case <-done:
+			break loop
+		default:
+			sampled += tr.sampleCount()
+		}
+	}
+	sampled += tr.sampleCount()
+	assert.Equal(t, int64(writers*perWriter), sampled)
+}