mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-15 05:29:28 +00:00
feat(metrics): instrument upstream latency
Add time-to-first-byte and total-duration histograms for every upstream call, observation only. No throttling, priorities, or control logic reads this data. The operator wants strain-based throttling later and needs real traffic data first. Extend watchdogBody (timeout.go) rather than add a new interception layer: it already wraps every upstream response body for provider adapters, native passthrough, and the MCP/A2A target proxy, and already observes the first successful read. armLatency arms a watchdogBody after construction, so newWatchdogBody keeps its existing signature and behavior; its own pinned tests still pass unchanged. A timestamp plus a few atomics are the only added cost; Read and Close each pay one nil check when latency collection is disabled. Expose llmgateway_upstream_ttfb_seconds and llmgateway_upstream_duration_seconds on the existing Prometheus endpoint (metrics.go). Both are in-process, per-replica histograms, never store-backed, so sum() is always the correct cross-replica aggregation. Buckets run 0.1s to 600s, chosen to resolve fast TTFB and short completions finely, cover the bulk of real chat completions, and bracket the five-minute default request timeout. Labels are provider and stream always, model only when Config.Metrics.ModelLabel is set, matching the existing per-model opt-in convention. Add a compact per-provider, per-stream average summary to GET /admin/api/overview, sourced from the same in-process store, without a full histogram or a per-model breakdown. Every write path is gated on metricsEnabled(cfg), so a deployment with metrics disabled collects nothing and the histogram families never render. No behavior change to request handling: same responses, status codes, headers, and accounting. Fix a Yaegi-only bug this feature's own context-threading change exposed: two distinct empty struct types used as context.WithValue keys compare equal to each other under the interpreter's interface equality, though never under compiled Go. Threading a second recorder (latencyRecorder) alongside the existing attemptRecorder made this observable for the first time, since compiled tests never see it and the plugin previously carried only one such key. Give latencyRecorderCtxKey a field so its size differs from attemptRecorderCtxKey's, confirmed against a minimal Yaegi reproduction before and after the fix. Fix a second bug, reproducible under compiled Go too: io.Reader may legally return its final data together with io.EOF in one call, and Go's real net/http body reader does this for small, fully-buffered responses. The original TTFB capture lived inside the err == nil branch, so it never fired for a typical small non-streaming completion. Move the check ahead of the error branch so it fires on any n > 0, leaving the pinned timer-reset and Close semantics untouched. Tests mutate the real source per case and confirm each one fails first, recorded in each test's own doc comment.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -551,6 +552,85 @@ func TestAdminOverview_ProviderRates_EndToEndAfterTraffic(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAdminOverview_LatencySummary_AvgReflectsSeededObservations proves
|
||||
// buildAdminLatencyViews' own math (admin.go, feat: instrument upstream
|
||||
// latency): plain averages (sum/count), split by stream state, and that
|
||||
// the opt-in per-model dimension never leaks into this compact admin
|
||||
// surface (the task brief's own "compact... do not bloat" requirement).
|
||||
//
|
||||
// alpha's observation is seeded directly via gw.latency.record under a
|
||||
// MODEL-keyed latencyKey only — bypassing recordLatency's own
|
||||
// dual-write (which always also writes the bare provider+stream key,
|
||||
// tested separately by metrics_test.go's TestMetrics_ModelLabel_*
|
||||
// family) — specifically so no base (model="") entry exists for
|
||||
// alpha's "streaming" state at all. That isolates buildAdminLatencyViews'
|
||||
// own model-exclusion filter: if it ever stopped skipping model-keyed
|
||||
// entries, THIS is the one that would leak through as a phantom row,
|
||||
// since there is no correctly-shaped base entry that could coincidentally
|
||||
// produce the same numbers instead (unlike a same-provider dual-write,
|
||||
// where the base and model entries are byte-identical and an overwrite
|
||||
// would go unnoticed).
|
||||
//
|
||||
// MUTATION VERIFIED: removing the `if s.key.model != "" ... continue`
|
||||
// guard from buildAdminLatencyViews (admin.go) made this test fail —
|
||||
// alpha.Latency gained a "streaming" entry (Count=1, AvgDurationMs=5000)
|
||||
// that must never exist, since alpha never received any base-key
|
||||
// observation. Reverted before committing.
|
||||
func TestAdminOverview_LatencySummary_AvgReflectsSeededObservations(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := newAdminTestConfig()
|
||||
h, gw := newAdminGatewayHandle(t, cfg)
|
||||
|
||||
// zeta, non-streaming: two base-key observations, 100ms and 300ms —
|
||||
// average 200ms.
|
||||
gw.recordLatency("zeta", "", latencySample{duration: 100 * time.Millisecond, ttfb: 100 * time.Millisecond, hasTTFB: true, streaming: false})
|
||||
gw.recordLatency("zeta", "", latencySample{duration: 300 * time.Millisecond, ttfb: 300 * time.Millisecond, hasTTFB: true, streaming: false})
|
||||
|
||||
// alpha, streaming: model-keyed only (see doc comment above).
|
||||
gw.latency.record(latencyKey{provider: "alpha", streaming: true, model: "a-model-1"},
|
||||
latencySample{duration: 5 * time.Second, ttfb: time.Second, hasTTFB: true, streaming: true})
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, adminRequest(http.MethodGet, adminOverviewPath, "sk-admin1"))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var got adminOverviewResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
var zeta, alpha *adminProviderView
|
||||
for i := range got.Providers {
|
||||
switch got.Providers[i].Name {
|
||||
case "zeta":
|
||||
zeta = &got.Providers[i]
|
||||
case "alpha":
|
||||
alpha = &got.Providers[i]
|
||||
}
|
||||
}
|
||||
if zeta == nil || alpha == nil {
|
||||
t.Fatalf("providers = %+v, want entries named zeta and alpha", got.Providers)
|
||||
}
|
||||
|
||||
nonStream, ok := zeta.Latency["non-streaming"]
|
||||
if !ok {
|
||||
t.Fatalf("zeta.Latency missing \"non-streaming\" entry: %+v", zeta.Latency)
|
||||
}
|
||||
if nonStream.Count != 2 {
|
||||
t.Errorf("non-streaming Count = %d, want 2", nonStream.Count)
|
||||
}
|
||||
if math.Abs(nonStream.AvgDurationMs-200) > 0.01 {
|
||||
t.Errorf("non-streaming AvgDurationMs = %v, want 200 ((100+300)/2)", nonStream.AvgDurationMs)
|
||||
}
|
||||
if math.Abs(nonStream.AvgTTFBMs-200) > 0.01 {
|
||||
t.Errorf("non-streaming AvgTTFBMs = %v, want 200", nonStream.AvgTTFBMs)
|
||||
}
|
||||
|
||||
if _, ok := alpha.Latency["streaming"]; ok {
|
||||
t.Errorf(`alpha.Latency has a "streaming" entry despite the only observation being model-keyed (never a base provider+stream one): %+v — the opt-in per-model dimension must never leak into this compact admin summary`, alpha.Latency)
|
||||
}
|
||||
}
|
||||
|
||||
// --- usage: math against seeded counters ---
|
||||
|
||||
func TestAdminUsage_MathAgainstSeededCounters(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user