feat(providers): success-rate accounting

Track per-provider and per-(provider,upstream-model) upstream-attempt
outcomes using the existing fixed-window counter machinery (limits.go),
following mcp_a2a.go's per-target counter pattern: kind-embedded keys
(kindProvider="prov", kindProviderModel="provmodel") keep a provider
scope collision-safe against a user/group/MCP-target/agent of the same
name.

limiter.recordProviderAttempt(provider, model, resp, err) increments an
"attempts" counter on every call, and a "failures" counter only for a
provider-fault outcome — reusing retry.go's own isTransient(resp, err)
classification exactly (transport/connection error, HTTP 429, or a 5xx),
not forking it. Both counters are written at minute and day granularity
only; provider health is a now-and-today question, not a billing one, so
there is no month window. limiter.providerUsage batches every scope's
read into one storeGetMulti round trip, mirroring targetUsage's own
single-batch discipline.

Wired at every chokepoint an upstream HTTP attempt resolves:
- providers.go's upstreamJSON/upstreamRawBytes path (chat, embeddings,
  images, audio) via retry.go's retryPolicy.do, which now looks up an
  optional attemptRecorder from ctx and invokes it once per retry
  attempt (attempt-accounting: a request retried twice before
  succeeding reports three attempts, not one).
- routes_unified.go's runUnified (chat/embeddings, streaming and
  non-streaming alike — both funnel through the same upstreamJSON call).
- routes_media.go's three media handlers (images.generations,
  audio.speech, audio.transcriptions).
- routes_passthrough.go's proxyUpstream (native provider passthrough) —
  a single un-retried attempt, provider-level only: the upstream model
  lives in the response body, read only after the attempt resolves.

The attemptRecorder is threaded via context (providers.go's
withAttemptRecorder/attemptRecorderFromContext), not a new function
parameter on every call site — proxyUpstream and retryPolicy.do are
both shared with paths that must NOT record provider accounting
(registry.go's discovery listModels calls, and the MCP/A2A target
proxy, mcp_a2a.go's handleTargetProxy), so scoping via context keeps
their signatures unchanged and those callers correctly opt out by
simply never wrapping their context.
This commit is contained in:
2026-08-21 23:22:16 +01:00
parent ad5adbcc1d
commit aafd2a0433
12 changed files with 653 additions and 4 deletions
+62
View File
@@ -480,6 +480,68 @@ func TestHandlePassthrough_NonStreamJSON_AccountsUsage(t *testing.T) {
if !ok || totalTokOut != 3 {
t.Errorf("total tokout/day counter = %d (ok=%v), want 3", totalTokOut, ok)
}
// Feature A (v0.22): handlePassthrough wraps r's context with an
// attemptRecorder before calling g.proxyUpstream — provider-level
// only (model="": the upstream model here lives in the response body,
// read only after this attempt already resolved, per
// recordProviderAttempt's own doc comment, limits.go).
attempts, ok := gw.limiter.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, time.Now())
if !ok || attempts != 1 {
t.Errorf("provider attempts/day = %d (ok=%v), want 1", attempts, ok)
}
fails, _ := gw.limiter.getCounter(kindProvider, "openai", metricProvFail, windowDay, time.Now())
if fails != 0 {
t.Errorf("provider fails/day = %d, want 0", fails)
}
}
// TestHandlePassthrough_DeadUpstream_RecordsProviderFailure proves a
// connection-level failure (proxyUpstream's client.Do returning a network
// error) still reports one provider-level attempt AND one failure —
// Feature A (v0.22): passthrough makes no retry.go attempt at all, so
// this exercises proxyUpstream's own attemptRecorderFromContext call
// directly, not retryPolicy.do's.
func TestHandlePassthrough_DeadUpstream_RecordsProviderFailure(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
deadURL := srv.URL
srv.Close() // closed before any request: every dial fails
cfg := CreateConfig()
cfg.Providers = map[string]*ProviderConfig{
"openai": {Type: "openai", BaseURL: deadURL, APIKey: "sk-up"},
}
cfg.Groups = map[string]*GroupConfig{"default": {}}
cfg.Users = &UsersConfig{Inline: []*UserConfig{
{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{}},
}}
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
h, err := New(context.Background(), next, cfg, "llmgw")
if err != nil {
t.Fatalf("New: %v", err)
}
gw, ok := h.(*Gateway)
if !ok {
t.Fatal("handler is not *Gateway")
}
req := httptest.NewRequest(http.MethodPost, "/openai/v1/native-endpoint", strings.NewReader(`{}`))
req.Header.Set("Authorization", "Bearer sk-alice")
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
if rec.Code != http.StatusBadGateway {
t.Fatalf("status = %d, want 502, body=%s", rec.Code, rec.Body.String())
}
attempts, ok := gw.limiter.getCounter(kindProvider, "openai", metricProvAttempt, windowDay, time.Now())
if !ok || attempts != 1 {
t.Errorf("provider attempts/day = %d (ok=%v), want 1", attempts, ok)
}
fails, ok := gw.limiter.getCounter(kindProvider, "openai", metricProvFail, windowDay, time.Now())
if !ok || fails != 1 {
t.Errorf("provider fails/day = %d (ok=%v), want 1", fails, ok)
}
}
// TestHandlePassthrough_GroupDeniesProvider_Returns403 covers a provider