feat(admin): add per-model usage statistics to the charts section

The Charts view could break usage down by user, group or total, but not by model. Per-model data existed only as day-window attempt and failure counts, which measure availability rather than delivered usage. A served model now accumulates the same four metrics a user or group does. The scope kind is "model" and its id is the canonical "provider/model", so a request that fails over counts against the provider that answered it, and one bare id served by two providers ranks as two rows. The request counter is written after the response, in the batch that already writes tokens and cost. That is the first point at which the serving model is known, so no extra round trip is needed and admission never double counts. GET /admin/api/usage/models ranks models for one metric and window. It returns only models with non-zero usage, by operator requirement: the catalog runs to hundreds of models and a padded ranking is unreadable. The read stays one storeGetMulti over the catalog. The Charts view gains a Models tab with its own metric picker, and the scope picker gains the models that have traffic.
2026-10-15 05:29:28 +00:00 · 2026-08-27 23:30:22 +01:00
parent bcd7c32c3d
commit 4bb49b4fe9
15 changed files with 1236 additions and 53 deletions
@@ -0,0 +1,150 @@
+// history.spec.ts covers useHistoryStore's model-ranking behaviour: the
+// Models tab reads a different endpoint from the other three tabs, and the
+// scope picker's model list is refreshed alongside whichever of the two is
+// active. Both must degrade independently — a failing picker list must
+// never blank a rendered chart or surface an error over it, which is the
+// asymmetry these tests pin.
+//
+// Same harness as dashboard.spec.ts: a Pinia store is plain reactive state
+// in the 'node' environment, with adminFetch (lib/api.ts) fully mocked, so
+// no real network or sessionStorage access happens.
+
+import { createPinia, setActivePinia } from 'pinia'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/api', async () => {
+  const actual = await vi.importActual<typeof import('@/lib/api')>('@/lib/api')
+  return { ...actual, adminFetch: vi.fn() }
+})
+
+import { adminFetch } from '@/lib/api'
+import { useAuthStore } from '@/stores/auth'
+import { metricsForTab, useHistoryStore } from '@/stores/history'
+import type { AdminUsageModelsResponse, UsageHistoryResponse } from '@/types/api'
+
+const mockedAdminFetch = vi.mocked(adminFetch)
+
+function modelsResponse(models: AdminUsageModelsResponse['models']): AdminUsageModelsResponse {
+  return { metric: 'cost', window: 'hour', models }
+}
+function seriesResponse(): UsageHistoryResponse {
+  return { scope: 'total', metric: 'req', window: 'hour', points: [{ bucket: '2026082012', value: 5 }] }
+}
+
+/** Routes a mocked adminFetch call by URL, so tests assert on behaviour rather than call order. */
+function routeFetch(handlers: { models?: () => unknown; series?: () => unknown }): void {
+  mockedAdminFetch.mockImplementation((path: string) => {
+    if (path.startsWith('/admin/api/usage/models')) {
+      return Promise.resolve(handlers.models ? handlers.models() : modelsResponse([]))
+    }
+    return Promise.resolve(handlers.series ? handlers.series() : seriesResponse())
+  })
+}
+
+describe('useHistoryStore model ranking', () => {
+  beforeEach(() => {
+    setActivePinia(createPinia())
+    mockedAdminFetch.mockReset()
+    useAuthStore().submit('test-admin-key')
+  })
+
+  it('asks the history endpoint for no metric on the Models tab', () => {
+    // The Models tab reads the ranking endpoint instead, so requesting a
+    // history metric for it would be a wasted round trip per refresh.
+    expect(metricsForTab('models')).toEqual([])
+    expect(metricsForTab('tokens')).toEqual(['tokin', 'tokout'])
+  })
+
+  it('fetches the ranking, not a series, when the Models tab is active', async () => {
+    routeFetch({ models: () => modelsResponse([{ id: 'uni/qwen3.8-flash-next', value: 4_100_000 }]) })
+    const history = useHistoryStore()
+    history.tab = 'models'
+
+    await history.refresh()
+
+    expect(history.modelRanking).toEqual([{ id: 'uni/qwen3.8-flash-next', value: 4_100_000 }])
+    const paths = mockedAdminFetch.mock.calls.map((c) => String(c[0]))
+    expect(paths.some((p) => p.startsWith('/admin/api/usage/history'))).toBe(false)
+  })
+
+  it('ranks by the selected metric, independently of the tab', async () => {
+    routeFetch({})
+    const history = useHistoryStore()
+    history.tab = 'models'
+
+    history.setModelMetric('tokin')
+    await history.refresh()
+
+    const rankingCall = mockedAdminFetch.mock.calls
+      .map((c) => String(c[0]))
+      .find((p) => p.startsWith('/admin/api/usage/models') && p.includes('metric=tokin'))
+    expect(rankingCall).toBeDefined()
+  })
+
+  it('refreshes the picker list alongside a time series', async () => {
+    routeFetch({ models: () => modelsResponse([{ id: 'alpha/a-model-1', value: 3 }]) })
+    const history = useHistoryStore()
+
+    await history.refresh()
+
+    // A model that has just started receiving traffic becomes selectable
+    // without a reload, even while a non-Models tab is on screen.
+    expect(history.modelOptions).toEqual([{ id: 'alpha/a-model-1', value: 3 }])
+    expect(history.seriesByMetric.req).toHaveLength(1)
+  })
+
+  it('asks the picker list for requests, so a free model is still selectable', async () => {
+    routeFetch({})
+    const history = useHistoryStore()
+
+    await history.fetchModelOptions()
+
+    const optionsCall = mockedAdminFetch.mock.calls.map((c) => String(c[0])).find((p) => p.includes('/usage/models'))
+    expect(optionsCall).toContain('metric=req')
+  })
+
+  it('swallows a picker-list failure without touching the chart error', async () => {
+    // Asserted directly on fetchModelOptions rather than through refresh:
+    // inside refresh a succeeding series fetch clears `error` afterwards
+    // anyway, which would let this pass even if the swallow were removed.
+    routeFetch({
+      models: () => {
+        throw new Error('models endpoint down')
+      },
+    })
+    const history = useHistoryStore()
+    history.error = 'pre-existing chart error'
+
+    await history.fetchModelOptions()
+
+    expect(history.error).toBe('pre-existing chart error')
+    expect(history.modelOptions).toEqual([])
+  })
+
+  it('keeps a rendered chart intact when the picker list fails', async () => {
+    routeFetch({
+      models: () => {
+        throw new Error('models endpoint down')
+      },
+    })
+    const history = useHistoryStore()
+
+    await history.refresh()
+
+    expect(history.seriesByMetric.req).toHaveLength(1)
+    expect(history.error).toBe('')
+  })
+
+  it('surfaces a failing ranking fetch, unlike the picker list', async () => {
+    mockedAdminFetch.mockImplementation((path: string) => {
+      if (path.includes('limit=20')) return Promise.reject(new Error('ranking unavailable'))
+      return Promise.resolve(modelsResponse([]))
+    })
+    const history = useHistoryStore()
+    history.tab = 'models'
+
+    await history.refresh()
+
+    expect(history.error).toBe('ranking unavailable')
+  })
+})
@@ -2,10 +2,61 @@ import { defineStore } from 'pinia'

 import { AdminApiError, adminFetch } from '@/lib/api'
 import { useAuthStore } from '@/stores/auth'
-import type { HistoryMetric, HistoryWindow, UsageHistoryPoint, UsageHistoryResponse } from '@/types/api'
+import type {
+  AdminUsageModelEntry,
+  AdminUsageModelsResponse,
+  HistoryMetric,
+  HistoryWindow,
+  UsageHistoryPoint,
+  UsageHistoryResponse,
+} from '@/types/api'

-/** The three chart tabs the Charts view offers — see metricsForTab below. */
-export type ChartTab = 'requests' | 'tokens' | 'cost'
+/**
+ * The chart tabs the Charts view offers. The first three are metrics over
+ * time (see metricsForTab below); "models" is the odd one out — a ranking
+ * across models for a single point in time, with its own metric picked
+ * separately (modelMetric below), because "which model" is the dimension
+ * that tab varies rather than the metric.
+ */
+export type ChartTab = 'requests' | 'tokens' | 'cost' | 'models'
+
+/**
+ * The tabs that render a time series — every ChartTab except the ranking.
+ * UsageChart is typed on this rather than on ChartTab so that adding
+ * another ranking-style tab is a compile error there, not a chart with a
+ * silently missing dataset spec.
+ */
+export type TimeSeriesTab = Exclude<ChartTab, 'models'>
+
+/**
+ * The metrics the Models ranking can rank by. Deliberately a single
+ * metric, unlike the stacked "tokens" time-series tab: a bar's LENGTH in a
+ * ranking is the comparison being made, so splitting it across two stacked
+ * series would make two models with different in/out mixes visually
+ * incomparable.
+ */
+export type ModelMetric = 'req' | 'tokin' | 'tokout' | 'cost'
+
+/** MODEL_METRIC_LABEL is the ranking metric picker's display text. */
+export const MODEL_METRIC_LABEL: Record<ModelMetric, string> = {
+  cost: 'Cost',
+  req: 'Requests',
+  tokin: 'Tokens in',
+  tokout: 'Tokens out',
+}
+
+/**
+ * How many ranked models the Models tab requests. Matches the server's own
+ * default (admin.go: usageModelsDefaultLimit) and stays well under its max.
+ */
+const MODEL_RANKING_LIMIT = 20
+
+/**
+ * How many models the SCOPE PICKER offers. Higher than the ranking's own
+ * limit — the picker is a lookup, not a top-N — and capped at the server's
+ * maximum (admin.go: usageModelsMaxLimit).
+ */
+const MODEL_OPTIONS_LIMIT = 100

 /**
 * How often the current selection refetches while the Charts view is open.
@@ -47,6 +98,10 @@ export function metricsForTab(tab: ChartTab): HistoryMetric[] {
      return ['tokin', 'tokout']
    case 'cost':
      return ['cost']
+    case 'models':
+      // The Models tab reads the ranking endpoint, not the history one —
+      // it needs no time series, so it asks for no metric here.
+      return []
  }
 }

@@ -63,7 +118,13 @@ export const useHistoryStore = defineStore('history', {
    scope: 'total',
    window: 'hour' as HistoryWindow,
    tab: 'requests' as ChartTab,
+    /** Which metric the Models ranking ranks by — independent of `tab`. */
+    modelMetric: 'cost' as ModelMetric,
    seriesByMetric: {} as Partial<Record<HistoryMetric, UsageHistoryPoint[]>>,
+    /** The current Models-tab ranking. Empty means "nothing used in this window". */
+    modelRanking: [] as AdminUsageModelEntry[],
+    /** Models with non-zero traffic, for the scope picker's model entries. */
+    modelOptions: [] as AdminUsageModelEntry[],
    loading: false,
    error: '',
    timer: undefined as ReturnType<typeof setInterval> | undefined,
@@ -72,17 +133,31 @@ export const useHistoryStore = defineStore('history', {
    setScope(scope: string): void {
      if (scope === this.scope) return
      this.scope = scope
-      void this.fetchSeries()
+      void this.refresh()
    },
    setWindow(window: HistoryWindow): void {
      if (window === this.window) return
      this.window = window
-      void this.fetchSeries()
+      void this.refresh()
    },
    setTab(tab: ChartTab): void {
      if (tab === this.tab) return
      this.tab = tab
-      void this.fetchSeries()
+      void this.refresh()
+    },
+    setModelMetric(metric: ModelMetric): void {
+      if (metric === this.modelMetric) return
+      this.modelMetric = metric
+      void this.refresh()
+    },
+    /**
+     * refresh fetches whatever the CURRENT selection needs: the ranking on
+     * the Models tab, a time series otherwise. The scope picker's model
+     * list is refreshed alongside either, so a model that has just started
+     * receiving traffic becomes selectable without a page reload.
+     */
+    async refresh(): Promise<void> {
+      await Promise.all([this.tab === 'models' ? this.fetchModelRanking() : this.fetchSeries(), this.fetchModelOptions()])
    },
    async fetchSeries(): Promise<void> {
      const auth = useAuthStore()
@@ -113,9 +188,49 @@ export const useHistoryStore = defineStore('history', {
        this.loading = false
      }
    },
+    async fetchModelRanking(): Promise<void> {
+      const auth = useAuthStore()
+      if (!auth.isAuthenticated) return
+      this.loading = true
+      try {
+        const res = await adminFetch<AdminUsageModelsResponse>(
+          `/admin/api/usage/models?metric=${this.modelMetric}&window=${this.window}&limit=${MODEL_RANKING_LIMIT}`,
+        )
+        this.modelRanking = res.models
+        this.error = ''
+      } catch (err) {
+        if (err instanceof AdminApiError && (err.status === 401 || err.status === 403)) {
+          return
+        }
+        this.error = err instanceof Error ? err.message : String(err)
+      } finally {
+        this.loading = false
+      }
+    },
+    /**
+     * fetchModelOptions loads the picker's model list: models with at least
+     * one REQUEST in the current window. Requests, not the ranking's own
+     * metric, deliberately — a model can serve traffic while reporting no
+     * tokens and costing nothing, and such a model must still be
+     * selectable. A failure here is swallowed rather than surfaced: it
+     * degrades the picker, and must not replace a rendered chart's own
+     * error (or clear it) on a background refresh.
+     */
+    async fetchModelOptions(): Promise<void> {
+      const auth = useAuthStore()
+      if (!auth.isAuthenticated) return
+      try {
+        const res = await adminFetch<AdminUsageModelsResponse>(
+          `/admin/api/usage/models?metric=req&window=${this.window}&limit=${MODEL_OPTIONS_LIMIT}`,
+        )
+        this.modelOptions = res.models
+      } catch {
+        // Intentionally ignored — see the doc comment above.
+      }
+    },
    startAutoRefresh(): void {
      if (this.timer !== undefined) return
-      this.timer = setInterval(() => void this.fetchSeries(), REFRESH_MS)
+      this.timer = setInterval(() => void this.refresh(), REFRESH_MS)
    },
    stopAutoRefresh(): void {
      if (this.timer === undefined) return