feat(admin): report per-model detail, including free models

GET /admin/api/usage/models accepts detail=1. Each entry then carries
requests, tokensIn, tokensOut, costMicroUsd and a free flag. A model is
listed when any of those four totals is non-zero, so a free model with
traffic appears with value 0 even when ranked by cost. The free flag is
true when modelMeta marks the model free, or its resolved price is zero.
Without detail the response is unchanged.

The image and audio routes now write the per-model request counter after
a successful upstream call, so media models show up in the ranking. They
still record no tokens or cost.

Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
2026-09-23 13:40:05 +01:00
parent b8f240caac
commit 3e05a79c0b
6 changed files with 823 additions and 8 deletions
+229 -7
View File
@@ -1223,6 +1223,36 @@ func parseUsageModelsSpan(raw, window string) (span int, ok bool) {
return n, true
}
// usageModelsDetailOn and usageModelsDetailOff enumerate GET
// /admin/api/usage/models' optional "detail" query parameter's accepted
// values (free-models feature): "1"/"true" turn detail on, ""/"0"/
// "false" leave it off (the byte-identical legacy response shape).
// Anything else is a 400, mirroring this endpoint's own
// validate-before-reading shape for every other parameter.
const (
usageModelsDetailOn1 = "1"
usageModelsDetailOn2 = "true"
usageModelsDetailOff0 = "0"
usageModelsDetailOff1 = "false"
usageModelsDetailOffRaw = ""
)
// parseUsageModelsDetail parses GET /admin/api/usage/models' optional
// "detail" query parameter: empty, "0" or "false" means off (today's
// byte-identical response — no Requests/TokensIn/TokensOut/CostMicroUSD/
// Free fields, no "detail" key at all thanks to its own omitempty); "1"
// or "true" means on. ok is false for anything else.
func parseUsageModelsDetail(raw string) (detail bool, ok bool) {
switch raw {
case usageModelsDetailOffRaw, usageModelsDetailOff0, usageModelsDetailOff1:
return false, true
case usageModelsDetailOn1, usageModelsDetailOn2:
return true, true
default:
return false, false
}
}
// catalogModelScopeIDs returns every catalogued (provider, model) pair's
// canonical kindModel scope id, in the registry snapshot's own order. It
// is the enumeration GET /admin/api/usage/models ranks over, and the same
@@ -1242,20 +1272,43 @@ func (g *Gateway) catalogModelScopeIDs() []string {
// adminUsageModelEntryView is one ranked model in GET
// /admin/api/usage/models: its canonical "provider/model" id and its
// total for the requested metric over the requested window's CURRENT
// bucket.
// bucket. detail=1 (free-models feature) additionally populates
// Requests/TokensIn/TokensOut/CostMicroUSD — pointers so a genuinely
// zero total is still emitted rather than indistinguishable from "not
// requested" — and Free. Every detail-only field is omitted (not
// zeroed) in the non-detail response, keeping it byte-identical to
// before this feature existed.
type adminUsageModelEntryView struct {
ID string `json:"id"`
Value int64 `json:"value"`
// Requests, TokensIn, TokensOut and CostMicroUSD are populated only
// by serveAdminUsageModelsDetail (detail=1); the non-detail path
// never sets them, so they stay nil and omitempty drops them.
// Ordered ahead of ID/Value/Free for fieldalignment (govet: minimal
// GC pointer-scan prefix) — JSON names are what's binding, not Go
// field order or the JSON key order it produces (the plan's own
// ruling, free-models plan's "Response" section).
Requests *int64 `json:"requests,omitempty"`
TokensIn *int64 `json:"tokensIn,omitempty"`
TokensOut *int64 `json:"tokensOut,omitempty"`
CostMicroUSD *int64 `json:"costMicroUsd,omitempty"`
ID string `json:"id"`
Value int64 `json:"value"`
// Free is populated only by serveAdminUsageModelsDetail too; false
// (the non-detail zero value) is dropped by its own omitempty.
Free bool `json:"free,omitempty"`
}
// adminUsageModelsResponse is GET /admin/api/usage/models' body. Span
// (F2, v0.3 dashboard task) echoes the resolved span (1 when the request
// carried no ?span= parameter — parseUsageModelsSpan's own default).
// Detail (free-models feature) echoes whether the request carried
// detail=1 (parseUsageModelsDetail) — omitted (false) via its own
// omitempty for the legacy, byte-identical detail=0/absent response.
type adminUsageModelsResponse struct {
Metric string `json:"metric"`
Window string `json:"window"`
Models []adminUsageModelEntryView `json:"models"`
Span int `json:"span"`
Detail bool `json:"detail,omitempty"`
}
// usageModelsChunkKeys bounds how many counterStore keys
@@ -1300,16 +1353,59 @@ func (g *Gateway) chunkedModelSpanTotals(ids []string, metric, window string, no
return out, true
}
// chunkedModelSpanTotalsMulti is chunkedModelSpanTotals generalized to
// several metrics read together (GET /admin/api/usage/models' detail=1
// mode, free-models feature: it needs all four metrics —
// req/tokin/tokout/cost — for every candidate id, not just the one the
// ranking sorts by). Each chunk's own key count is
// len(chunk)*span*len(metrics), so perChunk divides usageModelsChunkKeys
// by span*len(metrics) instead of span alone — chunkedModelSpanTotals'
// own reasoning, extended for the extra metrics dimension. now is
// resolved once by the caller and threaded through every chunk, exactly
// like chunkedModelSpanTotals, and any chunk that fails fails the whole
// call for the identical reason: a detail ranking silently missing an
// arbitrary subset of models is worse than a 503.
func (g *Gateway) chunkedModelSpanTotalsMulti(ids, metrics []string, window string, now time.Time, span int) ([][]int64, bool) {
if len(metrics) == 0 || span < 1 {
return nil, true
}
perChunk := usageModelsChunkKeys / (span * len(metrics))
if perChunk < 1 {
perChunk = 1
}
out := make([][]int64, len(metrics))
for m := range out {
out[m] = make([]int64, 0, len(ids))
}
for len(ids) > 0 {
n := perChunk
if n > len(ids) {
n = len(ids)
}
totals, ok := g.limiter.modelSpanTotalsMulti(ids[:n], metrics, window, now, span)
if !ok {
return nil, false
}
for m := range out {
out[m] = append(out[m], totals[m]...)
}
ids = ids[n:]
}
return out, true
}
// serveAdminUsageModels writes GET /admin/api/usage/models' ranking of the
// most-used models for one metric/window (the Charts view's "Models"
// tab): "metric"=req|tokin|tokout|cost, "window"=hour|day|month, optional
// "limit"=N (default and max per the constants above), optional
// "prefix"=STRING (item 6, v0.3 dashboard task round 2: the Charts
// view's model search box) narrowing the catalog to ids that start with
// it before anything is ranked — see below. Every parameter is validated
// before the store is touched — 400 for an unrecognized metric or
// window, an out-of-range limit, or a prefix over
// usageModelsPrefixMaxLen.
// it before anything is ranked — see below, optional "detail"=1|true
// (free-models feature; parseUsageModelsDetail) switching to
// serveAdminUsageModelsDetail below. Every parameter is validated before
// the store is touched — 400 for an unrecognized metric or window, an
// out-of-range limit, a prefix over usageModelsPrefixMaxLen, or an
// unrecognized detail value.
//
// ONLY NON-ZERO models are returned, by explicit operator requirement: the
// catalog runs to hundreds of models and a ranking padded with zeroes is
@@ -1334,6 +1430,13 @@ func (g *Gateway) chunkedModelSpanTotals(ids []string, metric, window string, no
// answers 503, never a silently-empty ranking — the same rule
// serveAdminUsageHistory applies, and for the same reason: "the store was
// unreachable" must never render as "no model was used".
//
// detail=0 (or an absent detail parameter) is byte-identical to this
// endpoint's behavior before the free-models feature existed — the
// branch below never runs, and adminUsageModelsResponse's own Detail
// field and every adminUsageModelEntryView detail-only field stay at
// their zero value, which their shared omitempty tag drops from the
// JSON entirely.
func (g *Gateway) serveAdminUsageModels(sw *statusTrackingWriter, r *http.Request) {
q := r.URL.Query()
@@ -1362,6 +1465,11 @@ func (g *Gateway) serveAdminUsageModels(sw *statusTrackingWriter, r *http.Reques
writeOAIError(sw, http.StatusBadRequest, "invalid_request_error", "invalid prefix")
return
}
detail, ok := parseUsageModelsDetail(q.Get("detail"))
if !ok {
writeOAIError(sw, http.StatusBadRequest, "invalid_request_error", "invalid detail")
return
}
ids := g.catalogModelScopeIDs()
if prefix != "" {
@@ -1373,6 +1481,12 @@ func (g *Gateway) serveAdminUsageModels(sw *statusTrackingWriter, r *http.Reques
}
ids = filtered
}
if detail {
g.serveAdminUsageModelsDetail(sw, ids, metric, window, span, limit)
return
}
totals, storeOK := g.chunkedModelSpanTotals(ids, metric, window, g.limiter.now(), span)
if !storeOK {
writeOAIError(sw, http.StatusServiceUnavailable, "server_error", "usage history store unavailable")
@@ -1400,6 +1514,114 @@ func (g *Gateway) serveAdminUsageModels(sw *statusTrackingWriter, r *http.Reques
_ = json.NewEncoder(sw).Encode(adminUsageModelsResponse{Metric: metric, Window: window, Span: span, Models: models})
}
// usageModelsDetailMetrics is the fixed metric read/unpack order
// serveAdminUsageModelsDetail uses: req, tokin, tokout, cost — matching
// adminUsageModelEntryView's own field order (Requests, TokensIn,
// TokensOut, CostMicroUSD). Built fresh per call rather than as a
// package-level var (go.md: no mutable package-level state) — the cost
// of one 4-element slice literal per detail request is immaterial next
// to the storeGetMulti round trip it drives.
func usageModelsDetailMetrics() []string {
return []string{metricReq, metricTokIn, metricTokOut, metricCost}
}
// usageModelsMetricIndex maps metric — already validated by
// validHistoryMetric, so always one of the four cases below — to its
// position in usageModelsDetailMetrics, picking out the "value" column
// GET /admin/api/usage/models' detail=1 mode ranks by: whichever metric
// the caller actually asked for, even though detail mode always reads
// all four.
func usageModelsMetricIndex(metric string) int {
switch metric {
case metricReq:
return 0
case metricTokIn:
return 1
case metricTokOut:
return 2
default: // metricCost: the only case validHistoryMetric still allows through
return 3
}
}
// adminUsageModelFree reports whether canonical (a catalogued
// "provider/model" id, bare its part after the first "/") is free for
// GET /admin/api/usage/models' detail=1 mode: either an operator
// modelMeta entry marks it Free (modelMetaFree, modelmeta.go — the SAME
// two-key lookup billing itself uses, so this view and actual billing
// can never disagree), or its resolved price (resolveUnifiedPricing,
// failover.go — the identical resolution unifiedCostMicros itself uses)
// is KNOWN and both per-token prices are exactly 0. The second check
// matters on its own: a configured or built-in price table entry that
// happens to bill nothing is just as "free" to an operator reading this
// ranking as an explicit modelMeta:{free:true}, but a genuinely UNKNOWN
// price must never render as free — resolveUnifiedPricing's own ok
// return keeps the two cases apart.
func adminUsageModelFree(canonical, bare string, overrides map[string]*ModelPricing, meta map[string]*ModelMetaConfig) bool {
if modelMetaFree(canonical, bare, meta) {
return true
}
price, known := resolveUnifiedPricing(canonical, bare, overrides)
return known && price.InputPerM == 0 && price.OutputPerM == 0
}
// serveAdminUsageModelsDetail is serveAdminUsageModels' detail=1 branch
// (free-models feature): ids (already prefix-filtered), metric, window,
// span and limit are all already validated by the caller. Unlike the
// non-detail path, an id is included when ANY of its four metrics is
// nonzero, not just the requested one — so a free model with real
// request traffic but zero cost still appears in a metric=cost ranking,
// at Value=0, instead of being dropped the way the non-detail path would
// drop it. Ties break on value desc, then requests desc, then id asc:
// once two free models tie on the requested metric (typically cost, both
// zero), requests is the natural next signal, ahead of the id fallback
// every ranking already uses to stay stable across polls.
func (g *Gateway) serveAdminUsageModelsDetail(sw *statusTrackingWriter, ids []string, metric, window string, span, limit int) {
metrics := usageModelsDetailMetrics()
totalsByMetric, storeOK := g.chunkedModelSpanTotalsMulti(ids, metrics, window, g.limiter.now(), span)
if !storeOK {
writeOAIError(sw, http.StatusServiceUnavailable, "server_error", "usage history store unavailable")
return
}
valueIdx := usageModelsMetricIndex(metric)
models := make([]adminUsageModelEntryView, 0, len(ids))
for i, id := range ids {
req := totalsByMetric[0][i]
tokIn := totalsByMetric[1][i]
tokOut := totalsByMetric[2][i]
cost := totalsByMetric[3][i]
if req == 0 && tokIn == 0 && tokOut == 0 && cost == 0 {
continue
}
_, bare, _ := strings.Cut(id, "/")
models = append(models, adminUsageModelEntryView{
ID: id,
Value: totalsByMetric[valueIdx][i],
Requests: &req,
TokensIn: &tokIn,
TokensOut: &tokOut,
CostMicroUSD: &cost,
Free: adminUsageModelFree(id, bare, g.cfg.Pricing, g.cfg.ModelMeta),
})
}
sort.Slice(models, func(i, j int) bool {
if models[i].Value != models[j].Value {
return models[i].Value > models[j].Value
}
if *models[i].Requests != *models[j].Requests {
return *models[i].Requests > *models[j].Requests
}
return models[i].ID < models[j].ID
})
if len(models) > limit {
models = models[:limit]
}
setAdminJSONHeaders(sw)
_ = json.NewEncoder(sw).Encode(adminUsageModelsResponse{Metric: metric, Window: window, Span: span, Models: models, Detail: true})
}
// parseEventsLimit parses GET /admin/api/events' "limit" query parameter
// (F3, v0.3 dashboard task): empty means adminEventsDefaultLimit,
// otherwise an integer between 1 and adminEventsMaxLimit inclusive,
+217
View File
@@ -2484,6 +2484,8 @@ func TestAdminUsageModels_ValidatesParameters(t *testing.T) {
{"limit not a number", "?metric=cost&window=day&limit=lots"},
// item 6, v0.3 dashboard task round 2: prefix over usageModelsPrefixMaxLen.
{"prefix over max length", "?metric=cost&window=day&prefix=" + strings.Repeat("a", usageModelsPrefixMaxLen+1)},
// free-models feature: detail must be "1"/"true"/""/"0"/"false", nothing else.
{"invalid detail", "?metric=cost&window=day&detail=yes"},
} {
t.Run(tc.name, func(t *testing.T) {
rec := httptest.NewRecorder()
@@ -2775,6 +2777,221 @@ func TestChunkedModelSpanTotals_SplitsIntoMultipleRoundTrips(t *testing.T) {
}
}
// TestChunkedModelSpanTotalsMulti_SplitsIntoMultipleRoundTrips is
// TestChunkedModelSpanTotals_SplitsIntoMultipleRoundTrips' own detail=1
// counterpart (free-models feature): a span- AND metrics-multiplied
// catalog read still stays chunked to at most usageModelsChunkKeys keys
// per storeGetMulti round trip. wantCalls is derived from the SAME
// perChunk/ceil-division formula chunkedModelSpanTotalsMulti itself
// applies (span*len(metrics) per id instead of span alone), so this
// asserts the real chunk boundary, not a hand-picked number.
func TestChunkedModelSpanTotalsMulti_SplitsIntoMultipleRoundTrips(t *testing.T) {
t.Parallel()
fixedNow := time.Date(2026, 9, 23, 10, 0, 0, 0, time.UTC)
const n = 200
const span = 6
metrics := []string{metricReq, metricTokIn, metricTokOut, metricCost}
store := &countingMultiStore{values: make(map[string]int64, n)}
ids := make([]string, n)
for i := 0; i < n; i++ {
ids[i] = fmt.Sprintf("provider/model-%d", i)
// Only the req metric's current bucket is seeded, and only with
// this id's own index — a direct probe of per-metric, per-id
// positional ordering across the chunk boundary.
store.values[windowKey(kindModel, ids[i], metricReq, windowDay, fixedNow)] = int64(i)
}
l := newLimiter(store, true)
l.nowFn = func() time.Time { return fixedNow }
gw := &Gateway{limiter: l}
got, ok := gw.chunkedModelSpanTotalsMulti(ids, metrics, windowDay, fixedNow, span)
if !ok {
t.Fatal("want ok=true")
}
if len(got) != len(metrics) {
t.Fatalf("len(got) = %d, want %d metric rows", len(got), len(metrics))
}
if len(got[0]) != n {
t.Fatalf("len(got[0]) = %d, want %d", len(got[0]), n)
}
perChunk := usageModelsChunkKeys / (span * len(metrics))
wantCalls := (n + perChunk - 1) / perChunk
if store.getMultiCalls != wantCalls {
t.Errorf("getMultiCalls = %d, want %d (%d ids, %d ids/chunk, each chunk <= %d keys = ids/chunk * span * len(metrics))", store.getMultiCalls, wantCalls, n, perChunk, usageModelsChunkKeys)
}
// req (metric index 0) preserves order/value across the chunk boundary.
if got[0][perChunk] != int64(perChunk) {
t.Errorf("got[0][%d] = %d, want %d (order preserved across the chunk boundary)", perChunk, got[0][perChunk], perChunk)
}
if got[0][n-1] != int64(n-1) {
t.Errorf("got[0][%d] = %d, want %d", n-1, got[0][n-1], n-1)
}
// Every other metric was never seeded, so its span sum must be 0.
for m := 1; m < len(metrics); m++ {
if got[m][0] != 0 {
t.Errorf("got[%d][0] = %d, want 0 (metric %q was never seeded)", m, got[m][0], metrics[m])
}
}
}
// --- usage/models: detail=1 (free-models feature) ---
// TestAdminUsageModels_Detail_OffKeepsLegacyByteShape proves detail=0,
// detail=false and an absent detail parameter all produce the exact same
// bytes — the endpoint's contract before this feature existed — and that
// none of the detail-only JSON keys ever appear in that shape.
func TestAdminUsageModels_Detail_OffKeepsLegacyByteShape(t *testing.T) {
t.Parallel()
cfg := newAdminTestConfig()
h, gw := newAdminGatewayHandle(t, cfg)
fixedNow := time.Date(2026, 8, 20, 12, 30, 0, 0, time.UTC)
gw.limiter.nowFn = func() time.Time { return fixedNow }
seedModelCounter(gw, "alpha/a-model-1", metricCost, windowDay, fixedNow, 900)
seedModelCounter(gw, "zeta/z-model", metricCost, windowDay, fixedNow, 4_100)
base := adminUsageModelsPath + "?metric=cost&window=day"
var bodies [][]byte
for _, query := range []string{base, base + "&detail=0", base + "&detail=false"} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, adminRequest(http.MethodGet, query, "sk-admin1"))
if rec.Code != http.StatusOK {
t.Fatalf("query=%s: status = %d, want 200, body=%s", query, rec.Code, rec.Body.String())
}
bodies = append(bodies, rec.Body.Bytes())
}
for i := 1; i < len(bodies); i++ {
if string(bodies[i]) != string(bodies[0]) {
t.Errorf("body for query %d = %q, want byte-identical to the absent-detail response %q", i, bodies[i], bodies[0])
}
}
body := string(bodies[0])
for _, key := range []string{`"detail"`, `"requests"`, `"tokensIn"`, `"tokensOut"`, `"costMicroUsd"`, `"free"`} {
if strings.Contains(body, key) {
t.Errorf("legacy (detail off) body unexpectedly contains %s: %s", key, body)
}
}
}
// TestAdminUsageModels_Detail_FreeModel_ZeroMetricStillIncluded_ViaModelMeta
// proves the problem this feature exists to fix: a free model with real
// request traffic but zero cost is invisible to the non-detail path
// (metric=cost drops any zero total) and visible, at Value=0, with
// Free=true, once detail=1 is set.
func TestAdminUsageModels_Detail_FreeModel_ZeroMetricStillIncluded_ViaModelMeta(t *testing.T) {
t.Parallel()
cfg := newAdminTestConfig()
cfg.ModelMeta = map[string]*ModelMetaConfig{
"alpha/a-model-1": {Free: true},
}
h, gw := newAdminGatewayHandle(t, cfg)
fixedNow := time.Date(2026, 8, 20, 12, 30, 0, 0, time.UTC)
gw.limiter.nowFn = func() time.Time { return fixedNow }
seedModelCounter(gw, "alpha/a-model-1", metricReq, windowDay, fixedNow, 5)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, adminRequest(http.MethodGet, adminUsageModelsPath+"?metric=cost&window=day&detail=1", "sk-admin1"))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var got adminUsageModelsResponse
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
if !got.Detail {
t.Error("Detail = false, want true")
}
if len(got.Models) != 1 || got.Models[0].ID != "alpha/a-model-1" {
t.Fatalf("models = %+v, want exactly alpha/a-model-1 (the only id with any nonzero metric)", got.Models)
}
entry := got.Models[0]
if entry.Value != 0 {
t.Errorf("Value = %d, want 0 (metric=cost, and cost is genuinely 0)", entry.Value)
}
if entry.Requests == nil || *entry.Requests != 5 {
t.Errorf("Requests = %v, want 5", entry.Requests)
}
if entry.CostMicroUSD == nil || *entry.CostMicroUSD != 0 {
t.Errorf("CostMicroUSD = %v, want a present 0, not omitted", entry.CostMicroUSD)
}
if !entry.Free {
t.Error("Free = false, want true (modelMeta Free:true)")
}
}
// TestAdminUsageModels_Detail_FreeViaZeroPriceOverride proves Free also
// follows from a KNOWN price of exactly 0 (a Pricing override, here),
// with no modelMeta entry involved at all — the plan's second free
// condition, distinct from modelMetaFree.
func TestAdminUsageModels_Detail_FreeViaZeroPriceOverride(t *testing.T) {
t.Parallel()
cfg := newAdminTestConfig()
cfg.Pricing = map[string]*ModelPricing{
"zeta/z-model": {InputPerM: 0, OutputPerM: 0},
}
h, gw := newAdminGatewayHandle(t, cfg)
fixedNow := time.Date(2026, 8, 20, 12, 30, 0, 0, time.UTC)
gw.limiter.nowFn = func() time.Time { return fixedNow }
seedModelCounter(gw, "zeta/z-model", metricReq, windowDay, fixedNow, 3)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, adminRequest(http.MethodGet, adminUsageModelsPath+"?metric=cost&window=day&detail=1", "sk-admin1"))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var got adminUsageModelsResponse
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
if len(got.Models) != 1 || got.Models[0].ID != "zeta/z-model" {
t.Fatalf("models = %+v, want exactly zeta/z-model", got.Models)
}
if !got.Models[0].Free {
t.Error("Free = false, want true (a known price of 0/0 must render as free)")
}
}
// TestAdminUsageModels_Detail_SortTieBreaksOnRequestsThenID proves the
// detail=1 sort order: value desc, THEN requests desc, THEN id asc. All
// three seeded models tie on cost (0); zeta/z-model's higher request
// count must rank it first despite "z" sorting after "a", and the
// remaining tie between the two alpha models falls through to id asc.
func TestAdminUsageModels_Detail_SortTieBreaksOnRequestsThenID(t *testing.T) {
t.Parallel()
cfg := newAdminTestConfig()
h, gw := newAdminGatewayHandle(t, cfg)
fixedNow := time.Date(2026, 8, 20, 12, 30, 0, 0, time.UTC)
gw.limiter.nowFn = func() time.Time { return fixedNow }
seedModelCounter(gw, "zeta/z-model", metricReq, windowDay, fixedNow, 10)
seedModelCounter(gw, "alpha/a-model-1", metricReq, windowDay, fixedNow, 5)
seedModelCounter(gw, "alpha/a-model-2", metricReq, windowDay, fixedNow, 5)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, adminRequest(http.MethodGet, adminUsageModelsPath+"?metric=cost&window=day&detail=1", "sk-admin1"))
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var got adminUsageModelsResponse
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("decode: %v", err)
}
wantOrder := []string{"zeta/z-model", "alpha/a-model-1", "alpha/a-model-2"}
if len(got.Models) != len(wantOrder) {
t.Fatalf("models = %+v, want %d entries", got.Models, len(wantOrder))
}
for i, wantID := range wantOrder {
if got.Models[i].ID != wantID {
t.Errorf("models[%d].ID = %q, want %q (full order = %+v)", i, got.Models[i].ID, wantID, got.Models)
}
}
}
// --- usage: rejectionsPerDay (F4, v0.3 dashboard task) ---
// TestAdminUsage_RejectionsPerDayField proves rejectionsPerDay is
+46
View File
@@ -2300,6 +2300,52 @@ func (l *limiter) modelTotals(ids []string, metric, window string) ([]int64, boo
return l.modelSpanTotals(ids, metric, window, l.now(), 1)
}
// modelSpanTotalsMulti is modelSpanTotals generalized to several metrics
// read in ONE storeGetMulti round trip — GET /admin/api/usage/models'
// detail=1 mode (free-models feature) needs all four metrics
// (req/tokin/tokout/cost) for every candidate id, not just the one the
// ranking sorts by, and a per-metric modelSpanTotals call would cost 4
// round trips instead of 1. Keys are built id-major, metric second, span
// innermost — out[m][i] is metrics[m]'s span-sum for ids[i], positional
// in both dimensions, mirroring modelSpanTotals' own result[i] contract.
// ok is false on any store read failure, exactly like modelSpanTotals —
// a detail read that only partially succeeds is never presented as
// complete data. An empty ids slice is not a store read at all, matching
// modelSpanTotals' own contract.
func (l *limiter) modelSpanTotalsMulti(ids, metrics []string, window string, now time.Time, span int) ([][]int64, bool) {
if len(ids) == 0 {
return nil, true
}
allKeys := make([]string, 0, len(ids)*len(metrics)*span)
for _, id := range ids {
for _, metric := range metrics {
keys, _ := historyBucketKeys(kindModel, id, metric, window, now, span)
allKeys = append(allKeys, keys...)
}
}
vals, ok := l.storeGetMulti(allKeys)
if !ok || len(vals) != len(allKeys) || l.configuredStoreDown() {
return nil, false
}
out := make([][]int64, len(metrics))
for m := range out {
out[m] = make([]int64, len(ids))
}
stride := len(metrics) * span
for i := range ids {
base := i * stride
for m := range metrics {
mBase := base + m*span
var sum int64
for _, v := range vals[mBase : mBase+span] {
sum += v
}
out[m][i] = sum
}
}
return out, true
}
// historyStepBack returns the instant window's bucket was current i steps
// before now: i=0 is now's own (current, possibly partial) bucket, i=1
// the previous one, and so on. hour and day step by a fixed duration/day
+36
View File
@@ -137,7 +137,9 @@ func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request
if _, err := adapter.imagesGeneration(ctx, sw, req); err != nil {
g.recordUpstreamEvent(scopes, upstreamModel, adapter.name(), routeImages, err)
g.handleAdapterError(sw, err, adapter.name())
return
}
g.recordMediaModelUsage(adapter.name(), upstreamModel)
}
// handleAudioSpeech implements POST /v1/audio/speech (spec §3): an
@@ -204,7 +206,9 @@ func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *u
if _, err := adapter.audioSpeech(ctx, sw, body, "application/json"); err != nil {
g.recordUpstreamEvent(scopes, upstreamModel, adapter.name(), routeAudioSpeech, err)
g.handleAdapterError(sw, err, adapter.name())
return
}
g.recordMediaModelUsage(adapter.name(), upstreamModel)
}
// handleAudioTranscriptions implements POST /v1/audio/transcriptions
@@ -310,7 +314,39 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
if _, err := adapter.audioTranscription(ctx, sw, uploadBody, uploadContentType); err != nil {
g.recordUpstreamEvent(scopes, upstreamModel, adapter.name(), routeAudioTranscriptions, err)
g.handleAdapterError(sw, err, adapter.name())
return
}
g.recordMediaModelUsage(adapter.name(), upstreamModel)
}
// recordMediaModelUsage writes ONLY the kindModel scope's own request
// counter for a media route (images/generations, audio/speech, audio/
// transcriptions) that has just been served successfully — a nil error
// from the adapter call means the upstream answered 2xx and its body was
// already forwarded to the client (openaiAdapter.imagesGeneration/
// audioSpeech/audioTranscription, provider_openai.go, translate a
// non-2xx upstream response into a non-nil error via
// newProviderHTTPError BEFORE writing anything to the client, and every
// other adapter's media methods either forward a genuine 2xx or return a
// *translateError with no upstream call at all — so err == nil is
// exactly "2xx was forwarded to the client" for all three routes).
//
// Unlike runUnified/handlePassthrough's own account call, this passes NO
// user/group/total scopes: admitRequestForRoute's checkAndCount already
// wrote their request counters at admission, and images/audio are never
// token- or cost-accounted (spec §3) — the only new counter a media
// route needs is the per-model request count GET
// /admin/api/usage/models ranks over (free-models feature), which
// nothing wrote for these routes before it. usage{} and cost 0 keep
// account's own tokin/tokout/cost writes skipped (account only writes a
// direction whose value is nonzero), so this is exactly one store batch
// containing the model scope's req counter at all three chart windows —
// the same single-round-trip account already batches for every other
// scope, called synchronously here, the same as every other account call
// site (routes_unified.go, routes_passthrough.go).
func (g *Gateway) recordMediaModelUsage(providerName, upstreamModel string) {
canonical := providerModelScopeID(providerName, upstreamModel)
g.limiter.account(withModelScope(nil, canonical), usage{}, 0)
}
// readAdmittedCapped claims a body-admission slot (acquireBodyAdmission),
+120
View File
@@ -1129,3 +1129,123 @@ func TestHandleImagesGenerations_LimitExceeded_429WithRetryAfter(t *testing.T) {
assert.Equal(t, http.StatusTooManyRequests, rec2.Code, "body=%s", rec2.Body.String())
assert.NotEmpty(t, rec2.Header().Get("Retry-After"))
}
// ---- recordMediaModelUsage: the per-model request counter (free-models feature) ----
// TestHandleImagesGenerations_RecordsMediaModelUsage proves
// recordMediaModelUsage (routes_media.go) writes the served model's own
// kindModel request counter on a successful (2xx) upstream call, and
// that a failed upstream call writes nothing at all for that scope — the
// same 2xx/failure boundary handleAdapterError's own err != nil branch
// already uses. images.generations is the representative route here,
// mirroring TestHandleImagesGenerations_UnknownModel_404/
// _LimitExceeded_429WithRetryAfter's own "one representative caller for
// a shared code path" convention above.
func TestHandleImagesGenerations_RecordsMediaModelUsage(t *testing.T) {
t.Run("2xx writes the model scope's req counter", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"created":1,"data":[{"b64_json":"xyz"}]}`))
}))
defer srv.Close()
gw := newMediaTestGateway(t, newMediaTestConfig(srv.URL, "img-test"))
body := map[string]any{"model": "img-test", "prompt": "a cat"}
req := newUnifiedRequest(t, http.MethodPost, imagesGenerationsPath, "sk-alice", body)
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
modelReq, ok := gw.limiter.getCounter(kindModel, "openai/img-test", metricReq, windowDay, time.Now())
assert.True(t, ok)
assert.Equal(t, int64(1), modelReq, "kindModel req counter")
// Only the model scope is new here — admission (checkAndCount)
// already wrote the user's own req counter, and this call must
// not add a second one on top of it.
userReq, ok := gw.limiter.getCounter("user", "alice", metricReq, windowMin, time.Now())
assert.True(t, ok)
assert.Equal(t, int64(1), userReq, "user scope req counter must still be exactly 1 (written once, by admission)")
// Images are never token/cost-accounted (spec §3) — the model
// scope's own tokin/tokout/cost must stay unwritten too, matching
// account's own "usage{}, cost 0 writes nothing beyond req" contract.
if tokIn, okIn := gw.limiter.getCounter(kindModel, "openai/img-test", metricTokIn, windowDay, time.Now()); okIn && tokIn != 0 {
t.Errorf("model scope tokin counter = %d, want 0", tokIn)
}
if cost, okCost := gw.limiter.getCounter(kindModel, "openai/img-test", metricCost, windowDay, time.Now()); okCost && cost != 0 {
t.Errorf("model scope cost counter = %d, want 0", cost)
}
})
t.Run("non-2xx writes no model scope counter at all", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer srv.Close()
gw := newMediaTestGateway(t, newMediaTestConfig(srv.URL, "img-test"))
body := map[string]any{"model": "img-test", "prompt": "a cat"}
req := newUnifiedRequest(t, http.MethodPost, imagesGenerationsPath, "sk-alice", body)
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.NotEqual(t, http.StatusOK, rec.Code, "the upstream 500 must surface as a client-visible error, not 200")
modelReq, ok := gw.limiter.getCounter(kindModel, "openai/img-test", metricReq, windowDay, time.Now())
if ok && modelReq != 0 {
t.Errorf("model scope req counter = %d, want 0 (a failed upstream call must write no model-scope counter)", modelReq)
}
})
}
// TestHandleAudioSpeech_RecordsMediaModelUsage is
// TestHandleImagesGenerations_RecordsMediaModelUsage's audio/speech
// counterpart, proving the same model-scope counter lands through the
// binary-streaming forward path (openaiAdapter.audioSpeech) too.
func TestHandleAudioSpeech_RecordsMediaModelUsage(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "audio/mpeg")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("fake-audio-bytes"))
}))
defer srv.Close()
gw := newMediaTestGateway(t, newMediaTestConfig(srv.URL, "tts-test"))
body := map[string]any{"model": "tts-test", "input": "hello", "voice": "alloy"}
req := newUnifiedRequest(t, http.MethodPost, audioSpeechPath, "sk-alice", body)
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
modelReq, ok := gw.limiter.getCounter(kindModel, "openai/tts-test", metricReq, windowDay, time.Now())
assert.True(t, ok)
assert.Equal(t, int64(1), modelReq, "kindModel req counter")
}
// TestHandleAudioTranscriptions_RecordsMediaModelUsage is
// TestHandleImagesGenerations_RecordsMediaModelUsage's audio/
// transcriptions counterpart, proving the same model-scope counter lands
// through the multipart-replay forward path
// (openaiAdapter.audioTranscription) too.
func TestHandleAudioTranscriptions_RecordsMediaModelUsage(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"text":"hello world"}`))
}))
defer srv.Close()
gw := newMediaTestGateway(t, newMediaTestConfig(srv.URL, "whisper-test"))
origBody, contentType := buildMultipartTranscription(t, "whisper-test")
req := httptest.NewRequest(http.MethodPost, audioTranscriptionsPath, bytes.NewReader(origBody))
req.Header.Set("Content-Type", contentType)
req.Header.Set("Authorization", "Bearer sk-alice")
rec := httptest.NewRecorder()
gw.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code, "body=%s", rec.Body.String())
modelReq, ok := gw.limiter.getCounter(kindModel, "openai/whisper-test", metricReq, windowDay, time.Now())
assert.True(t, ok)
assert.Equal(t, int64(1), modelReq, "kindModel req counter")
}
+175 -1
View File
@@ -762,7 +762,18 @@ func run() error {
// builtinModelMetaTable map literal, proving that specific path
// runs correctly under Yaegi too, not just the config-override
// one testDataWantModel already exercises.
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
// provenanceEstProviderModel additionally carries a modelMeta
// free:true override (free-models feature) — it has no
// configured price at all (unifiedCostMicros already bills it 0
// via the "unknown price, warn" path, exerciseUsageProvenance's
// own assertions above never check its cost value, so this adds
// no regression there), so free:true here is the ONLY reason
// adminUsageModelFree (admin.go) marks it free: exactly the
// modelMeta path exerciseFreeModelUsageDetail (below) exists to
// exercise interpreted, as opposed to the OTHER free path (a
// KNOWN price of exactly 0), which carries no fixture of its own
// here — both are already covered compiled (admin_test.go).
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000},"` + provenanceEstProviderModel + `":{"free":true}},` +
`"mcpServers":{"` + mcpProbeServerName + `":{"url":"` + mcpProbeUpstream.URL + `"},"` + targetHealthDownServerName + `":{"url":"` + targetHealthDownURL + `"},"` + targetHealthProbeOnlyServerName + `":{"url":"` + targetHealthProbeOnlyUpstream.URL + `"}},` +
// targetHealth (feat/target-health): enabled. probeInterval is
// deliberately well ABOVE the floor (10s): exerciseTargetHealthProbeWarmup
@@ -960,6 +971,21 @@ func run() error {
if err := exerciseModelUsageRanking(handler); err != nil {
return err
}
// exerciseFreeModelUsageDetail/exerciseMediaModelUsage (free-models
// feature) run right after: the first only READS admin endpoints
// (detail=1), the second sends one real POST /v1/images/generations
// that adds exactly 1 to modelUsageCanonicalID's own req counter — a
// perturbation exerciseModelUsageRanking's own assertions (above,
// already run) never re-check, and nothing below depends on that
// counter's exact value either (exerciseUsageModelsSpan only checks
// the echoed span; exerciseRateLimitEventAndRejections uses its own
// dedicated principal).
if err := exerciseFreeModelUsageDetail(handler); err != nil {
return err
}
if err := exerciseMediaModelUsage(handler); err != nil {
return err
}
// exerciseAdminOverviewReplicaAndWarnings/exerciseUsageModelsSpan (F4/
// F9/F10/F2, v0.3 dashboard task) run next: both only READ admin
// endpoints, so neither perturbs nor is perturbed by anything above.
@@ -3445,6 +3471,154 @@ func pollModelRanking(handler http.Handler) ([]any, error) {
}
}
// freeModelDetailCanonicalID is the kindModel scope id
// exerciseFreeModelUsageDetail expects to find, with Free=true, in GET
// /admin/api/usage/models' detail=1 response: provenanceEstProviderModel
// already carries one real request (exerciseUsageProvenance's own chat
// completion, above) but no configured price at all — see its own
// modelMeta:{"free":true} override's doc comment (attemptAccountingOverride)
// for why free:true there is the ONLY reason this shows up free here.
const freeModelDetailCanonicalID = provenanceEstProviderName + "/" + provenanceEstProviderModel
// pollUsageModelsDetail is pollModelRanking's own detail=1 counterpart
// (free-models feature): GET /admin/api/usage/models?metric=cost&window=day&detail=1,
// polled the identical way — a single immediate read can outrun the
// accounting write an immediately preceding request triggered, under the
// slower interpreted path.
func pollUsageModelsDetail(handler http.Handler) ([]any, error) {
deadline := time.Now().Add(2 * time.Second)
for {
req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=cost&window=day&detail=1", nil)
req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
return nil, fmt.Errorf("GET /admin/api/usage/models?detail=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
}
var body map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
return nil, fmt.Errorf("decode GET /admin/api/usage/models?detail=1 body: %w", err)
}
models, ok := body["models"].([]any)
if !ok {
return nil, fmt.Errorf("admin usage/models?detail=1 body has no models array: %s", rec.Body.String())
}
if len(models) > 0 || time.Now().After(deadline) {
return models, nil
}
time.Sleep(10 * time.Millisecond)
}
}
// exerciseFreeModelUsageDetail drives GET /admin/api/usage/models'
// detail=1 mode (free-models feature) under the INTERPRETER: a
// modelMeta-declared free model with real request traffic but a
// genuinely unpriced (hence zero-cost) model must still appear in a
// metric=cost ranking once detail=1 is set — the non-detail path drops
// any zero-value entry entirely, which is exactly the gap this feature
// closes. Reached through serveAdminUsageModelsDetail's own sort.Slice
// closure, its pointer-typed struct fields, and adminUsageModelFree's
// two-source free() check — none of which any earlier probe in this
// harness exercises.
func exerciseFreeModelUsageDetail(handler http.Handler) error {
models, err := pollUsageModelsDetail(handler)
if err != nil {
return err
}
for _, raw := range models {
entry, ok := raw.(map[string]any)
if !ok {
return fmt.Errorf("GET /admin/api/usage/models?detail=1: entry is not an object: %v", raw)
}
if id, _ := entry["id"].(string); id != freeModelDetailCanonicalID {
continue
}
value, _ := entry["value"].(float64)
if value != 0 {
return fmt.Errorf("detail=1 entry %q: value = %v, want 0 (metric=cost, and its price is genuinely unknown)", freeModelDetailCanonicalID, value)
}
free, _ := entry["free"].(bool)
if !free {
return fmt.Errorf("detail=1 entry %q: free = %v, want true (modelMeta free:true override)", freeModelDetailCanonicalID, entry["free"])
}
requests, _ := entry["requests"].(float64)
if requests <= 0 {
return fmt.Errorf("detail=1 entry %q: requests = %v, want > 0", freeModelDetailCanonicalID, entry["requests"])
}
fmt.Println("yaegi-check: GET /admin/api/usage/models detail=1 lists a modelMeta-free model at value 0 with its real request count, under the interpreter")
return nil
}
return fmt.Errorf("GET /admin/api/usage/models?detail=1 did not list %q: %v", freeModelDetailCanonicalID, models)
}
// modelUsageDetailRequests reads GET /admin/api/usage/models' detail=1
// "requests" count for id right now. No poll: account()'s own store
// write (limits.go) is synchronous — unlike recordProviderAttempt, it
// never goes through limiter.spawn — so by the time ServeHTTP returns
// for the request that triggered it, the counter this reads is already
// current; pollUsageModelsDetail's own poll loop exists for a DIFFERENT
// reason (an empty response, not a stale one) and still applies here
// unchanged. An id with no traffic at all yet is legitimately absent
// from the response (only a nonzero metric total is listed even under
// detail=1), so a miss reports 0, not an error.
func modelUsageDetailRequests(handler http.Handler, id string) (int64, error) {
models, err := pollUsageModelsDetail(handler)
if err != nil {
return 0, err
}
for _, raw := range models {
entry, ok := raw.(map[string]any)
if !ok {
continue
}
if entry["id"] != id {
continue
}
reqs, _ := entry["requests"].(float64)
return int64(reqs), nil
}
return 0, nil
}
// exerciseMediaModelUsage drives POST /v1/images/generations (free-
// models feature: media-route model request counters) under the
// INTERPRETER, reusing the existing "openai"/testDataWantModel provider/
// model pair — upstream answers 200 for any request path (its own doc
// comment, above), so no new fixture provider is needed here. It reads
// modelUsageCanonicalID's own kindModel req counter (via GET
// /admin/api/usage/models detail=1) before and after the request:
// recordMediaModelUsage (routes_media.go) must have added exactly 1,
// reached through withModelScope called with an empty scope list — a
// construct no earlier media-route probe in this harness exercises.
func exerciseMediaModelUsage(handler http.Handler) error {
before, err := modelUsageDetailRequests(handler, modelUsageCanonicalID)
if err != nil {
return err
}
imgReq := httptest.NewRequest(http.MethodPost, "/v1/images/generations", strings.NewReader(
`{"model":"`+testDataWantModel+`","prompt":"a cat"}`,
))
imgReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
imgReq.Header.Set("Content-Type", "application/json")
imgRec := httptest.NewRecorder()
handler.ServeHTTP(imgRec, imgReq)
if imgRec.Code != http.StatusOK {
return fmt.Errorf("POST /v1/images/generations (media-model-usage harness): status = %d, want 200, body=%s", imgRec.Code, imgRec.Body.String())
}
after, err := modelUsageDetailRequests(handler, modelUsageCanonicalID)
if err != nil {
return err
}
if after != before+1 {
return fmt.Errorf("GET /admin/api/usage/models detail=1 requests for %q = %d, want exactly %d (before=%d, one served media request must add exactly one)", modelUsageCanonicalID, after, before+1, before)
}
fmt.Println("yaegi-check: a served media route (POST /v1/images/generations) incremented its model's own per-model request counter, under the interpreter")
return nil
}
// exerciseAdminOverviewReplicaAndWarnings proves GET /admin/api/overview's
// two new fields (F4/F9 replica id, F10 config-warnings collection, v0.3
// dashboard task) are populated correctly under the REAL interpreter: