merge: show upstream latency in the admin Providers tab

webui/src/components/ProvidersView.vue

@@ -21,11 +21,11 @@ import {
   CardTitle,
 } from '@/components/ui/card'
 import { useSearchQuery } from '@/composables/useSearchQuery'
-import { formatAgo, formatContextWindow, formatUntil, refreshDetailLabel, refreshLabel, routableModelId } from '@/lib/format'
+import { formatAgo, formatContextWindow, formatLatencyMs, formatUntil, refreshDetailLabel, refreshLabel, routableModelId } from '@/lib/format'
 import { isModelDegraded } from '@/lib/provider-rate'
 import { type ExpandState, clearExpandOverrides, computeExpandedItems, toggleItemExpand } from '@/lib/search-expand'
 import { useDashboardStore } from '@/stores/dashboard'
-import type { AdminAliasView, AdminModelMetaView, AdminModelRateView, AdminProviderView } from '@/types/api'
+import type { AdminAliasView, AdminLatencyView, AdminModelMetaView, AdminModelRateView, AdminProviderView } from '@/types/api'
 
 // This component backs the "Providers" tab (App.vue) — a UI-label rename
 // only. The data it renders still comes from GET /admin/api/overview
@@ -199,6 +199,58 @@ function healthDetailClass(state: AdminProviderView['healthState']): string {
   return state === 'open' ? 'text-destructive' : 'text-muted-foreground'
 }
 
+// --- upstream latency (feat: instrument upstream latency) ---
+//
+// Beside the discovery-health badge above: no badge at all for a stream
+// state with no observations yet (p.latency undefined, or missing that
+// key) — mirrors healthBadgeVariant/modelIsDegraded's own "nothing to
+// show reads as no badge" convention, so absent data reads as absent,
+// never as a fabricated zero.
+//
+// The two stream states are deliberately labeled and worded differently,
+// never merged into one figure (task brief's own correctness rule):
+// streaming shows avgTtfbMs, the load-sensitive reading, labeled "ttfb";
+// non-streaming shows avgDurationMs labeled "avg" — NEVER "latency" or
+// "ttfb" — because a non-streaming provider buffers its whole completion
+// before sending anything, so its own avgTtfbMs is approximately equal
+// to avgDurationMs and carries the identical output-length contamination
+// a raw total-duration figure always has (a 4000-token answer legitimately
+// takes longer than a 50-token one on an equally healthy provider).
+
+/** streamingLatency/nonStreamingLatency pull p.latency's two known stream-state keys (admin.go's buildAdminLatencyViews emits exactly these two strings) — undefined when that stream state has no observations yet. */
+function streamingLatency(p: AdminProviderView): AdminLatencyView | undefined {
+  return p.latency?.streaming
+}
+function nonStreamingLatency(p: AdminProviderView): AdminLatencyView | undefined {
+  return p.latency?.['non-streaming']
+}
+
+/** latencyObservationCount renders v.count (omitted, per admin.go, when 0) as a singular/plural detail phrase for a badge title, falling back to "no observations counted" for the defensive case of a present view with an absent count (version skew). */
+function latencyObservationCount(v: AdminLatencyView): string {
+  if (v.count === undefined) return 'no observations counted'
+  return `${v.count} ${v.count === 1 ? 'observation' : 'observations'} on this replica`
+}
+
+/**
+ * PER_REPLICA_CAVEAT is appended to every latency badge's title (task
+ * brief's correctness rule 1): this figure comes from the same
+ * in-process accumulator the rate-limit rejection counter reads, never
+ * Redis, so it reflects only whichever of this deployment's several
+ * Traefik replicas answered the current poll — never a fleet-wide
+ * average, and a freshly restarted pod shows a short window.
+ */
+const PER_REPLICA_CAVEAT = 'Per-replica, in-process only — not a fleet-wide average across this deployment’s replicas.'
+
+/** streamingLatencyTitle is the streaming ttfb badge's hover/title detail. */
+function streamingLatencyTitle(v: AdminLatencyView): string {
+  return `Average time to first byte, the load-sensitive reading for streaming traffic. ${latencyObservationCount(v)}. ${PER_REPLICA_CAVEAT}`
+}
+
+/** nonStreamingLatencyTitle is the non-streaming avg-duration badge's hover/title detail — explicit that this is NOT a latency/ttfb reading (correctness rule 2). */
+function nonStreamingLatencyTitle(v: AdminLatencyView): string {
+  return `Average total response time, including generation — not a load-sensitive signal like streaming ttfb, since this provider buffers the whole completion before sending anything. ${latencyObservationCount(v)}. ${PER_REPLICA_CAVEAT}`
+}
+
 // --- model aliases (sortable DataTable, operator directive) ---
 //
 // The alias id gets the ModelChip copy treatment (an alias name IS the
@@ -340,6 +392,33 @@ const aliasEmptyMessage = computed(() =>
                 >
                   {{ healthBadgeLabel(p) }}
                 </Badge>
+                <!--
+                  Upstream latency (feat: instrument upstream latency):
+                  streaming's ttfb is the load-sensitive figure; the
+                  non-streaming badge deliberately says "avg", never
+                  "ttfb"/"latency" — see nonStreamingLatencyTitle's own
+                  doc comment for why that number is not a load signal.
+                  No badge at all for a stream state with no observations
+                  yet, mirroring the health badge's own convention above.
+                -->
+                <Badge
+                  v-if="streamingLatency(p)?.avgTtfbMs !== undefined"
+                  as="span"
+                  variant="outline"
+                  class="font-normal tabular-nums"
+                  :title="streamingLatencyTitle(streamingLatency(p) as AdminLatencyView)"
+                >
+                  streaming ttfb {{ formatLatencyMs(streamingLatency(p)!.avgTtfbMs as number) }}
+                </Badge>
+                <Badge
+                  v-if="nonStreamingLatency(p)?.avgDurationMs !== undefined"
+                  as="span"
+                  variant="outline"
+                  class="font-normal tabular-nums"
+                  :title="nonStreamingLatencyTitle(nonStreamingLatency(p) as AdminLatencyView)"
+                >
+                  non-streaming avg {{ formatLatencyMs(nonStreamingLatency(p)!.avgDurationMs as number) }}
+                </Badge>
                 <FontAwesomeIcon
                   v-if="p.lastErr"
                   :icon="faTriangleExclamation"

webui/src/lib/format.spec.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest'
 
-import { formatCompactCount, formatContextWindow, formatModelCostHover, formatUntil, refreshDetailLabel, refreshLabel } from './format'
+import { formatCompactCount, formatContextWindow, formatLatencyMs, formatModelCostHover, formatUntil, refreshDetailLabel, refreshLabel } from './format'
 
 const ZERO_TIME = '0001-01-01T00:00:00Z'
 
@@ -164,3 +164,22 @@ describe('formatCompactCount', () => {
     expect(formatCompactCount(-15_000_000)).toBe('-15M')
   })
 })
+
+// Latency panel (feat: instrument upstream latency, webui surface):
+// avgTtfbMs/avgDurationMs arrive from admin.go as plain millisecond
+// floats. formatLatencyMs is deliberately a plain rounding, not a
+// floor-not-round boundary rule like formatCompactCount above — these are
+// averages with no discrete threshold a reader could misinterpret as
+// crossed.
+describe('formatLatencyMs', () => {
+  it.each([
+    { ms: 0, want: '0ms' },
+    { ms: 42.7, want: '43ms' }, // sub-1000ms rounds to the nearest ms
+    { ms: 999.4, want: '999ms' }, // just under the 1000ms/1s boundary
+    { ms: 1000, want: '1.0s' }, // the boundary itself switches to seconds
+    { ms: 1234, want: '1.2s' },
+    { ms: 12_345, want: '12.3s' },
+  ])('formatLatencyMs($ms) = $want', ({ ms, want }) => {
+    expect(formatLatencyMs(ms)).toBe(want)
+  })
+})

webui/src/lib/format.ts

@@ -147,6 +147,21 @@ export function formatModelCostHover(inputUsd: number, outputUsd: number): strin
   return `in ${formatUsdPerMTok(inputUsd)} / out ${formatUsdPerMTok(outputUsd)} per MTok`
 }
 
+/**
+ * formatLatencyMs renders a millisecond average (admin.go's
+ * adminLatencyView.avgTtfbMs/avgDurationMs — feat: latency panel) as a
+ * compact duration: plain milliseconds below 1000 ("120ms"), one-decimal
+ * seconds at or above it ("1.2s"). These values are already averages
+ * (sum/count from the in-process latency accumulator), so rounding here
+ * carries none of formatCompactCount's boundary-crossing honesty concern
+ * above — there is no discrete threshold a caller could misread as
+ * crossed, just a plain rounded reading.
+ */
+export function formatLatencyMs(ms: number): string {
+  if (ms < 1000) return `${Math.round(ms)}ms`
+  return `${(ms / 1000).toFixed(1)}s`
+}
+
 /** formatAgo renders an ISO timestamp as "(Ns ago)", or "" for the unset zero-time sentinel. */
 export function formatAgo(iso: string | undefined): string {
   if (!iso || iso === ZERO_TIME) return ''

webui/src/types/api.ts

@@ -40,6 +40,24 @@ export interface AdminModelMetaView {
   outputPerMTokUsd?: number
 }
 
+/**
+ * One provider's one stream-state's compact latency summary (admin.go:
+ * adminLatencyView — feat: instrument upstream latency), sourced from the
+ * SAME in-process g.latency accumulator /metrics reads, never Redis —
+ * PER-REPLICA, not fleet-wide (see AdminProviderView.latency's own doc
+ * comment). Every field is optional/undefined, not a plain 0: each is
+ * `omitempty` on the Go side, so undefined means "no observations for
+ * this field", never a fabricated zero. avgTtfbMs in particular must
+ * never be treated as interchangeable with avgDurationMs — see
+ * AdminProviderView.latency's doc comment for why they diverge for
+ * non-streaming traffic.
+ */
+export interface AdminLatencyView {
+  avgTtfbMs?: number
+  avgDurationMs?: number
+  count?: number
+}
+
 export interface AdminProviderView {
   name: string
   type: string
@@ -96,6 +114,34 @@ export interface AdminProviderView {
   failuresMinute: number
   /** Per-model breakdown of the counters above, keyed by upstream model id — always present (possibly empty), never omitted. */
   modelRates: Record<string, AdminModelRateView>
+  /**
+   * latency is this provider's compact per-stream average TTFB/duration
+   * summary (admin.go: adminProviderView.Latency — feat: instrument
+   * upstream latency), keyed by "streaming"/"non-streaming" — the same
+   * two string keys buildAdminLatencyViews (admin.go) emits, mirrored
+   * here as a literal union rather than a bare Record<string, ...> so a
+   * caller cannot accidentally index a third, nonexistent stream state.
+   * Undefined entirely for a provider with no observations yet, or a
+   * deployment with metrics collection gated off — undefined, never an
+   * empty object, so "no data" is never confused with "data says zero".
+   *
+   * PER-REPLICA, IN-PROCESS: sourced from the same in-process
+   * accumulator the rate-limit rejection counters read, never Redis.
+   * This deployment runs multiple Traefik replicas, so any single poll
+   * reflects only whichever replica answered it, and a freshly restarted
+   * pod shows a short observation window — never render this as if it
+   * were a fleet-wide average.
+   *
+   * STREAMING VS NON-STREAMING ARE NOT COMPARABLE: avgTtfbMs is the
+   * load-sensitive figure, and only for stream === 'streaming' — a
+   * non-streaming provider buffers its whole completion before sending
+   * anything, so its own avgTtfbMs is approximately equal to its own
+   * avgDurationMs and carries the identical output-length contamination
+   * (a long completion legitimately takes longer than a short one on an
+   * equally healthy provider). Never average the two stream states
+   * together or present either one as a bare, unlabeled "latency".
+   */
+  latency?: Partial<Record<'streaming' | 'non-streaming', AdminLatencyView>>
 }
 
 export interface AdminRedisView {