feat(admin): add per-model usage statistics to the charts section

The Charts view could break usage down by user, group or total, but not
by model. Per-model data existed only as day-window attempt and failure
counts, which measure availability rather than delivered usage.

A served model now accumulates the same four metrics a user or group
does. The scope kind is "model" and its id is the canonical
"provider/model", so a request that fails over counts against the
provider that answered it, and one bare id served by two providers ranks
as two rows.

The request counter is written after the response, in the batch that
already writes tokens and cost. That is the first point at which the
serving model is known, so no extra round trip is needed and admission
never double counts.

GET /admin/api/usage/models ranks models for one metric and window. It
returns only models with non-zero usage, by operator requirement: the
catalog runs to hundreds of models and a padded ranking is unreadable.
The read stays one storeGetMulti over the catalog.

The Charts view gains a Models tab with its own metric picker, and the
scope picker gains the models that have traffic.
This commit is contained in:
2026-08-27 23:30:22 +01:00
parent bcd7c32c3d
commit 4bb49b4fe9
15 changed files with 1236 additions and 53 deletions
+144 -1
View File
@@ -40,6 +40,7 @@ import (
"math"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"reflect"
@@ -693,7 +694,15 @@ func run() error {
// runs LAST, for the identical reason: it reads /metrics and GET
// /admin/api/overview one more time and must not perturb any counter
// an earlier probe already asserted on.
return exerciseUsageProvenance(handler)
if err := exerciseUsageProvenance(handler); err != nil {
return err
}
// exerciseModelUsageRanking (feat: per-model usage statistics) runs
// after everything above: it only READS admin endpoints plus one more
// chat completion of its own, and asserts on a kindModel counter no
// earlier probe touches, so it can neither perturb nor be perturbed by
// the provider-level counters they assert on.
return exerciseModelUsageRanking(handler)
}
// builtinLookupModelID is a real, stable entry in the generated
@@ -2415,3 +2424,137 @@ func parseScalar(s string) any {
}
return s
}
// modelUsageCanonicalID is the kindModel scope id exerciseModelUsageRanking
// expects its own chat completion to be attributed to: the canonical
// "provider/model" of the bare testDataWantModel's winning provider
// ("openai" — the same attribution exerciseAttemptAccounting already
// relies on).
const modelUsageCanonicalID = "openai/" + testDataWantModel
// exerciseModelUsageRanking drives the per-model usage feature end to end
// under the INTERPRETER (feat: per-model usage statistics): one real chat
// completion, then GET /admin/api/usage/models for the ranking and GET
// /admin/api/usage/history for that same model's own series.
//
// Why this needs an interpreted probe at all, when compiled tests already
// cover the same code: the feature adds a scope kind resolved POST-response
// (limits.go's withModelScope/kindModel), reached through a sort.Slice
// closure and a generic JSON encode of a newly declared struct type. Every
// one of those is a construct this repo has previously seen diverge under
// Yaegi — a compiled pass proves nothing about the interpreter, which is
// the whole premise of this harness.
func exerciseModelUsageRanking(handler http.Handler) error {
chatReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
`{"model":"`+testDataWantModel+`","messages":[{"role":"user","content":"hi"}]}`,
))
chatReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
chatReq.Header.Set("Content-Type", "application/json")
chatRec := httptest.NewRecorder()
handler.ServeHTTP(chatRec, chatReq)
if chatRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/chat/completions (model-usage harness): status = %d, want 200, body=%s", chatRec.Code, chatRec.Body.String())
}
models, err := pollModelRanking(handler)
if err != nil {
return err
}
// The ranking must contain the model that just served, and EVERY entry
// must be non-zero — the operator requirement the server-side filter
// exists for. A zero here would mean the filter did not run under the
// interpreter even though it does when compiled.
var found bool
for _, raw := range models {
entry, ok := raw.(map[string]any)
if !ok {
return fmt.Errorf("GET /admin/api/usage/models: entry is not an object: %v", raw)
}
id, _ := entry["id"].(string)
value, _ := entry["value"].(float64)
if value == 0 {
return fmt.Errorf("GET /admin/api/usage/models listed %q at value 0 — only non-zero models may be returned", id)
}
if id == modelUsageCanonicalID {
found = true
}
}
if !found {
return fmt.Errorf("GET /admin/api/usage/models did not list %q after a served request: %v", modelUsageCanonicalID, models)
}
// The same model must also be reachable as an ordinary history scope —
// the point of keying model usage as a scope kind rather than a
// bespoke counter.
histReq := httptest.NewRequest(http.MethodGet,
"/admin/api/usage/history?scope=model:"+url.QueryEscape(modelUsageCanonicalID)+"&metric=req&window=day&span=1", nil)
histReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
histRec := httptest.NewRecorder()
handler.ServeHTTP(histRec, histReq)
if histRec.Code != http.StatusOK {
return fmt.Errorf("GET /admin/api/usage/history for scope model:%s: status = %d, want 200, body=%s",
modelUsageCanonicalID, histRec.Code, histRec.Body.String())
}
var hist map[string]any
if err := json.Unmarshal(histRec.Body.Bytes(), &hist); err != nil {
return fmt.Errorf("decode model history body: %w", err)
}
points, ok := hist["points"].([]any)
if !ok || len(points) != 1 {
return fmt.Errorf("model history points = %v, want exactly 1 bucket: %s", hist["points"], histRec.Body.String())
}
point, ok := points[0].(map[string]any)
if !ok {
return fmt.Errorf("model history point is not an object: %v", points[0])
}
if value, _ := point["value"].(float64); value < 1 {
return fmt.Errorf("model history value = %v, want at least 1 request attributed to %q", point["value"], modelUsageCanonicalID)
}
// An id outside the catalog must 404 rather than answer an empty
// series — the guard that keeps a mistyped or retired model from
// looking like a model with no traffic.
unknownReq := httptest.NewRequest(http.MethodGet,
"/admin/api/usage/history?scope=model:openai/no-such-model&metric=req&window=day&span=1", nil)
unknownReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
unknownRec := httptest.NewRecorder()
handler.ServeHTTP(unknownRec, unknownReq)
if unknownRec.Code != http.StatusNotFound {
return fmt.Errorf("GET /admin/api/usage/history for an uncatalogued model: status = %d, want 404, body=%s",
unknownRec.Code, unknownRec.Body.String())
}
fmt.Println("yaegi-check: per-model usage ranking and model-scoped history served under the interpreter; only non-zero models listed, uncatalogued model 404s")
return nil
}
// pollModelRanking reads GET /admin/api/usage/models until it reports at
// least one model or a 2s budget elapses. The poll mirrors
// pollProviderAttemptCounters' own rationale: interpreted code driving the
// request runs slower than compiled code, so a single immediate read can
// outrun the accounting write the preceding request triggered.
func pollModelRanking(handler http.Handler) ([]any, error) {
deadline := time.Now().Add(2 * time.Second)
for {
req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=day", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return nil, fmt.Errorf("GET /admin/api/usage/models: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
return nil, fmt.Errorf("decode GET /admin/api/usage/models body: %w", err)
}
models, ok := body["models"].([]any)
if !ok {
return nil, fmt.Errorf("admin usage/models body has no models array: %s", rec.Body.String())
}
if len(models) > 0 || time.Now().After(deadline) {
return models, nil
}
time.Sleep(10 * time.Millisecond)
}
}