From 0f78b98192366c2c5e2a1271dec8ca196989fa88 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 23 Aug 2026 19:46:33 +0100 Subject: [PATCH] test(yaegi-check): verify usage-provenance metrics under the interpreter Add a provenance-est provider whose upstream answers with no "usage" field at all, driving a real, non-streaming chat completion against it under Yaegi. exerciseUsageProvenance then parses GET /metrics with the real Prometheus TextParser and confirms both new families (llmgateway_usage_provenance_requests_total and -_tokens_total) render as COUNTER type with a distinct "estimated" series for this provider and a distinct "reported" series for openai's own already-driven traffic, then confirms GET /admin/api/overview surfaces the identical estimated observation and never leaks a "reported" key into the compact admin summary. Run at the end of run(), after every other probe that already reads /metrics or /admin/api/overview, so it never perturbs a counter an earlier probe asserted on. --- tools/yaegi-check/main.go | 206 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 8 deletions(-) diff --git a/tools/yaegi-check/main.go b/tools/yaegi-check/main.go index 8688cf3..09d88c4 100644 --- a/tools/yaegi-check/main.go +++ b/tools/yaegi-check/main.go @@ -164,6 +164,26 @@ const ( hungBodyUpstreamSleep = 600 * time.Millisecond ) +// provenanceEstProviderName/provenanceEstProviderModel back +// exerciseUsageProvenance (feat: expose token-accounting provenance): a +// provider whose upstream answers 200 with NO "usage" field at all — +// decodes to chatUsagePayload's zero value (provider_openai.go's +// forwardJSON), the exact shape runUnified's own estimation fallback +// (routes_unified.go) classifies as "estimated" — proving the +// substitution math (ceil(len(body)/4)) and the new +// llmgateway_usage_provenance_requests_total/-_tokens_total families +// (metrics.go) all run correctly under the REAL interpreter, not merely +// compiled. "reported" is already exercised, under the interpreter, by +// exerciseAttemptAccounting's own real, usage-bearing "openai" traffic +// above — no separate fixture needed for that provenance here. +const ( + provenanceEstProviderName = "provenance-est" + // provenanceEstProviderModel: same reasoning as timeoutProviderModel's + // own doc comment above — must not collide with testDataWantModel or + // any other probe's model id. + provenanceEstProviderModel = "gpt-provenance-est-test" +) + // excludedTopLevelDirs lists repo-root directories the GOPATH copy must // never include: build tooling, integration fixtures, planning docs and // VCS metadata have nothing to do with the plugin package Yaegi imports. @@ -348,6 +368,18 @@ func run() error { })) defer hungBodyUpstream.Close() + // provenanceEstUpstream backs exerciseUsageProvenance (feat: expose + // token-accounting provenance): answers 200 with NO "usage" field at + // all — see provenanceEstProviderName's own const block doc comment + // for why this is the exact shape runUnified's estimation fallback + // classifies as "estimated". + provenanceEstUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"id":"c-prov","object":"chat.completion","model":"` + provenanceEstProviderModel + `","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`)) + })) + defer provenanceEstUpstream.Close() + // mcpProbeUpstream answers federation's outbound tools/call with a // body deliberately larger than mcpBackendCallResponseMaxBytes, so // exerciseHandler's POST /mcp probe below drives doBackendJSONRPC's @@ -492,6 +524,9 @@ func run() error { // comment above for why this provider exists alongside // timeoutProviderName. `"` + hungBodyProviderName + `":{"type":"openai","baseUrl":"` + hungBodyUpstream.URL + `","apiKey":"sk-up","requestTimeout":"` + hungBodyProviderRequestTimeout + `","models":["` + hungBodyProviderModel + `"]},` + + // provenanceEstProviderName (feat: expose token-accounting + // provenance) — see its own const block doc comment above. + `"` + provenanceEstProviderName + `":{"type":"openai","baseUrl":"` + provenanceEstUpstream.URL + `","apiKey":"sk-up","models":["` + provenanceEstProviderModel + `"]},` + // "anthropic" backs the /v1/messages passthrough probes // (exerciseMessagesRoute, below): a real anthropic-type provider, // interpreted end to end through @@ -591,14 +626,21 @@ func run() error { return err } // exerciseLatencyMetrics (feat: instrument upstream latency) runs - // LAST, for the same reason exerciseMetricsRoute itself used to be - // last: it reads the exact same /metrics endpoint and must not race - // or perturb anything exerciseBreaker/exerciseMetricsRoute still care - // about. It relies on exerciseHandler's own exerciseAttemptAccounting - // sub-probe (already run, above) having driven a real, non-streaming - // POST /v1/chat/completions against the "openai" provider — see its - // own doc comment. - return exerciseLatencyMetrics(handler) + // after exerciseMetricsRoute, for the same reason exerciseMetricsRoute + // itself used to be last: it reads the exact same /metrics endpoint + // and must not race or perturb anything exerciseBreaker/ + // exerciseMetricsRoute still care about. It relies on exerciseHandler's + // own exerciseAttemptAccounting sub-probe (already run, above) having + // driven a real, non-streaming POST /v1/chat/completions against the + // "openai" provider — see its own doc comment. + if err := exerciseLatencyMetrics(handler); err != nil { + return err + } + // exerciseUsageProvenance (feat: expose token-accounting provenance) + // runs LAST, for the identical reason: it reads /metrics and GET + // /admin/api/overview one more time and must not perturb any counter + // an earlier probe already asserted on. + return exerciseUsageProvenance(handler) } // builtinLookupModelID is a real, stable entry in the generated @@ -1035,6 +1077,154 @@ func exerciseLatencyMetrics(handler http.Handler) error { return nil } +// exerciseUsageProvenance proves the usage-provenance metric surface +// (feat: expose token-accounting provenance, metrics.go's +// provenanceStore/writeProvenanceMetrics, admin.go's +// buildAdminProvenanceViews) renders correctly under the REAL +// interpreter — not merely that a compiled `go test` accepts the same +// bytes, this package's own doc comment's exact concern. +// +// Drives one real, non-streaming POST /v1/chat/completions against +// provenanceEstProviderName, whose upstream (provenanceEstUpstream, run()) +// answers 200 with no "usage" field at all — the exact shape +// runUnified's estimation fallback (routes_unified.go) classifies as +// "estimated". "reported" is not driven fresh here: exerciseHandler's own +// exerciseAttemptAccounting sub-probe (already run, above) already drove +// a real, usage-bearing POST /v1/chat/completions against "openai", +// which is reported provenance by construction — reusing that traffic +// instead of a third fixture, mirroring exerciseLatencyMetrics' own +// "no new traffic" reasoning for its own duration-histogram assertion. +func exerciseUsageProvenance(handler http.Handler) error { + chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader( + `{"model":"`+provenanceEstProviderName+`/`+provenanceEstProviderModel+`","messages":[{"role":"user","content":"hi"}]}`, + )) + chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey) + chatReq.Header.Set("Content-Type", "application/json") + chatRec := httptest.NewRecorder() + handler.ServeHTTP(chatRec, chatReq) + if chatRec.Code != http.StatusOK { + return fmt.Errorf("POST /v1/chat/completions (usage-provenance harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String()) + } + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.RemoteAddr = metricsProbeOutsideAddr + req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + return fmt.Errorf("GET /metrics (usage-provenance probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String()) + } + + parser := expfmt.NewTextParser(model.LegacyValidation) + families, err := parser.TextToMetricFamilies(strings.NewReader(rec.Body.String())) + if err != nil { + return fmt.Errorf("GET /metrics: a real Prometheus TextParser rejected the interpreted rendering: %w (body=%s)", err, rec.Body.String()) + } + + requestsFamily, ok := families["llmgateway_usage_provenance_requests_total"] + if !ok { + return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_requests_total family in the parsed output (body=%s)", rec.Body.String()) + } + if requestsFamily.GetType() != dto.MetricType_COUNTER { + return fmt.Errorf("llmgateway_usage_provenance_requests_total type = %s, want COUNTER", requestsFamily.GetType()) + } + tokensFamily, ok := families["llmgateway_usage_provenance_tokens_total"] + if !ok { + return fmt.Errorf("GET /metrics: no llmgateway_usage_provenance_tokens_total family in the parsed output (body=%s)", rec.Body.String()) + } + if tokensFamily.GetType() != dto.MetricType_COUNTER { + return fmt.Errorf("llmgateway_usage_provenance_tokens_total type = %s, want COUNTER", tokensFamily.GetType()) + } + + findSample := func(fam *dto.MetricFamily, provider, provenance string) *dto.Metric { + for _, m := range fam.GetMetric() { + var p, k string + for _, lp := range m.GetLabel() { + switch lp.GetName() { + case "provider": + p = lp.GetValue() + case "provenance": + k = lp.GetValue() + } + } + if p == provider && k == provenance { + return m + } + } + return nil + } + + estReq := findSample(requestsFamily, provenanceEstProviderName, "estimated") + if estReq == nil { + return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String()) + } + if estReq.GetCounter().GetValue() != 1 { + return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider=%q,provenance="estimated"} = %v, want 1`, provenanceEstProviderName, estReq.GetCounter().GetValue()) + } + estTok := findSample(tokensFamily, provenanceEstProviderName, "estimated") + if estTok == nil { + return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} sample found (body=%s)`, provenanceEstProviderName, rec.Body.String()) + } + if estTok.GetCounter().GetValue() <= 0 { + return fmt.Errorf(`llmgateway_usage_provenance_tokens_total{provider=%q,provenance="estimated"} = %v, want > 0 (estimated substitutes a nonzero prompt count)`, provenanceEstProviderName, estTok.GetCounter().GetValue()) + } + + reportedReq := findSample(requestsFamily, "openai", "reported") + if reportedReq == nil { + return fmt.Errorf(`GET /metrics: no llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} sample found (body=%s) — exerciseAttemptAccounting's own earlier real, usage-bearing traffic must have been classified "reported"`, rec.Body.String()) + } + if reportedReq.GetCounter().GetValue() < 1 { + return fmt.Errorf(`llmgateway_usage_provenance_requests_total{provider="openai",provenance="reported"} = %v, want >= 1`, reportedReq.GetCounter().GetValue()) + } + + // GET /admin/api/overview: proves admin.go's buildAdminProvenanceViews + // wiring also runs correctly interpreted, decoded generically (this + // harness module is compiled, not interpreted; adminOverviewResponse/ + // adminProviderView are unexported — see readProviderAttemptCounters' + // own doc comment for why a generic decode is simpler here than + // exporting test-only types across that boundary). + adminReq := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil) + adminReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey) + adminRec := httptest.NewRecorder() + handler.ServeHTTP(adminRec, adminReq) + if adminRec.Code != http.StatusOK { + return fmt.Errorf("GET /admin/api/overview (usage-provenance probe): status = %d, want 200, body=%s", adminRec.Code, adminRec.Body.String()) + } + var overview map[string]any + if err := json.Unmarshal(adminRec.Body.Bytes(), &overview); err != nil { + return fmt.Errorf("decode GET /admin/api/overview body: %w", err) + } + providers, _ := overview["providers"].([]any) + var provenanceEstProvider map[string]any + for _, p := range providers { + pm, ok := p.(map[string]any) + if !ok { + continue + } + if pm["name"] == provenanceEstProviderName { + provenanceEstProvider = pm + break + } + } + if provenanceEstProvider == nil { + return fmt.Errorf("GET /admin/api/overview: no provider entry named %q (body=%s)", provenanceEstProviderName, adminRec.Body.String()) + } + provenanceField, _ := provenanceEstProvider["provenance"].(map[string]any) + estimatedEntry, _ := provenanceField["estimated"].(map[string]any) + if estimatedEntry == nil { + return fmt.Errorf(`GET /admin/api/overview: provider %q has no provenance.estimated entry: %+v`, provenanceEstProviderName, provenanceField) + } + if reqs, _ := estimatedEntry["requests"].(float64); reqs != 1 { + return fmt.Errorf("GET /admin/api/overview: provider %q provenance.estimated.requests = %v, want 1", provenanceEstProviderName, estimatedEntry["requests"]) + } + if _, hasReported := provenanceField["reported"]; hasReported { + return fmt.Errorf(`GET /admin/api/overview: provider %q provenance has a "reported" key, want it excluded from this compact admin summary: %+v`, provenanceEstProviderName, provenanceField) + } + + fmt.Println("yaegi-check: usage-provenance metrics (llmgateway_usage_provenance_requests_total/-_tokens_total) and admin overview parsed by a real Prometheus TextParser/JSON decoder; estimated and reported provenances both present and distinct") + return nil +} + // exerciseHandler runs two real requests against the interpreted // handler, drawn from the plugin's own .traefik.yml testData: an // authenticated GET /v1/models, which must return the configured