From 3012586e8edc66821af288e73f8a1803b9efd74e Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Sun, 23 Aug 2026 18:55:16 +0100 Subject: [PATCH] feat(webui): show per-replica upstream latency in the Providers tab Add streaming and non-streaming latency badges beside the discovery health badge on each provider row. Streaming shows average time to first byte, labeled ttfb, since it is the only load-sensitive figure this data provides. Non-streaming shows average total duration, labeled avg, never latency or ttfb, because a non-streaming provider buffers the whole completion before sending anything and its ttfb carries the same output-length contamination as its duration. Badges omit entirely when a provider has no observations yet, so absence never reads as a fabricated zero. Every badge title states the per-replica, in-process scope explicitly: this data comes from the same in-process accumulator the rate-limit counters use, never Redis, so a single poll reflects only whichever replica answered it. Mirror admin.go's adminLatencyView type in types/api.ts, keyed by the literal streaming/non-streaming union admin.go emits. Add formatLatencyMs to lib/format.ts with spec tests alongside its siblings in format.spec.ts. --- webui/src/components/ProvidersView.vue | 83 +++++++++++++++++++++++++- webui/src/lib/format.spec.ts | 21 ++++++- webui/src/lib/format.ts | 15 +++++ webui/src/types/api.ts | 46 ++++++++++++++ 4 files changed, 162 insertions(+), 3 deletions(-) diff --git a/webui/src/components/ProvidersView.vue b/webui/src/components/ProvidersView.vue index b3ff21d..664f501 100644 --- a/webui/src/components/ProvidersView.vue +++ b/webui/src/components/ProvidersView.vue @@ -21,11 +21,11 @@ import { CardTitle, } from '@/components/ui/card' import { useSearchQuery } from '@/composables/useSearchQuery' -import { formatAgo, formatContextWindow, formatUntil, refreshDetailLabel, refreshLabel, routableModelId } from '@/lib/format' +import { formatAgo, formatContextWindow, formatLatencyMs, formatUntil, refreshDetailLabel, refreshLabel, routableModelId } from '@/lib/format' import { isModelDegraded } from '@/lib/provider-rate' import { type ExpandState, clearExpandOverrides, computeExpandedItems, toggleItemExpand } from '@/lib/search-expand' import { useDashboardStore } from '@/stores/dashboard' -import type { AdminAliasView, AdminModelMetaView, AdminModelRateView, AdminProviderView } from '@/types/api' +import type { AdminAliasView, AdminLatencyView, AdminModelMetaView, AdminModelRateView, AdminProviderView } from '@/types/api' // This component backs the "Providers" tab (App.vue) — a UI-label rename // only. The data it renders still comes from GET /admin/api/overview @@ -199,6 +199,58 @@ function healthDetailClass(state: AdminProviderView['healthState']): string { return state === 'open' ? 'text-destructive' : 'text-muted-foreground' } +// --- upstream latency (feat: instrument upstream latency) --- +// +// Beside the discovery-health badge above: no badge at all for a stream +// state with no observations yet (p.latency undefined, or missing that +// key) — mirrors healthBadgeVariant/modelIsDegraded's own "nothing to +// show reads as no badge" convention, so absent data reads as absent, +// never as a fabricated zero. +// +// The two stream states are deliberately labeled and worded differently, +// never merged into one figure (task brief's own correctness rule): +// streaming shows avgTtfbMs, the load-sensitive reading, labeled "ttfb"; +// non-streaming shows avgDurationMs labeled "avg" — NEVER "latency" or +// "ttfb" — because a non-streaming provider buffers its whole completion +// before sending anything, so its own avgTtfbMs is approximately equal +// to avgDurationMs and carries the identical output-length contamination +// a raw total-duration figure always has (a 4000-token answer legitimately +// takes longer than a 50-token one on an equally healthy provider). + +/** streamingLatency/nonStreamingLatency pull p.latency's two known stream-state keys (admin.go's buildAdminLatencyViews emits exactly these two strings) — undefined when that stream state has no observations yet. */ +function streamingLatency(p: AdminProviderView): AdminLatencyView | undefined { + return p.latency?.streaming +} +function nonStreamingLatency(p: AdminProviderView): AdminLatencyView | undefined { + return p.latency?.['non-streaming'] +} + +/** latencyObservationCount renders v.count (omitted, per admin.go, when 0) as a singular/plural detail phrase for a badge title, falling back to "no observations counted" for the defensive case of a present view with an absent count (version skew). */ +function latencyObservationCount(v: AdminLatencyView): string { + if (v.count === undefined) return 'no observations counted' + return `${v.count} ${v.count === 1 ? 'observation' : 'observations'} on this replica` +} + +/** + * PER_REPLICA_CAVEAT is appended to every latency badge's title (task + * brief's correctness rule 1): this figure comes from the same + * in-process accumulator the rate-limit rejection counter reads, never + * Redis, so it reflects only whichever of this deployment's several + * Traefik replicas answered the current poll — never a fleet-wide + * average, and a freshly restarted pod shows a short window. + */ +const PER_REPLICA_CAVEAT = 'Per-replica, in-process only — not a fleet-wide average across this deployment’s replicas.' + +/** streamingLatencyTitle is the streaming ttfb badge's hover/title detail. */ +function streamingLatencyTitle(v: AdminLatencyView): string { + return `Average time to first byte, the load-sensitive reading for streaming traffic. ${latencyObservationCount(v)}. ${PER_REPLICA_CAVEAT}` +} + +/** nonStreamingLatencyTitle is the non-streaming avg-duration badge's hover/title detail — explicit that this is NOT a latency/ttfb reading (correctness rule 2). */ +function nonStreamingLatencyTitle(v: AdminLatencyView): string { + return `Average total response time, including generation — not a load-sensitive signal like streaming ttfb, since this provider buffers the whole completion before sending anything. ${latencyObservationCount(v)}. ${PER_REPLICA_CAVEAT}` +} + // --- model aliases (sortable DataTable, operator directive) --- // // The alias id gets the ModelChip copy treatment (an alias name IS the @@ -340,6 +392,33 @@ const aliasEmptyMessage = computed(() => > {{ healthBadgeLabel(p) }} + + + streaming ttfb {{ formatLatencyMs(streamingLatency(p)!.avgTtfbMs as number) }} + + + non-streaming avg {{ formatLatencyMs(nonStreamingLatency(p)!.avgDurationMs as number) }} + { expect(formatCompactCount(-15_000_000)).toBe('-15M') }) }) + +// Latency panel (feat: instrument upstream latency, webui surface): +// avgTtfbMs/avgDurationMs arrive from admin.go as plain millisecond +// floats. formatLatencyMs is deliberately a plain rounding, not a +// floor-not-round boundary rule like formatCompactCount above — these are +// averages with no discrete threshold a reader could misinterpret as +// crossed. +describe('formatLatencyMs', () => { + it.each([ + { ms: 0, want: '0ms' }, + { ms: 42.7, want: '43ms' }, // sub-1000ms rounds to the nearest ms + { ms: 999.4, want: '999ms' }, // just under the 1000ms/1s boundary + { ms: 1000, want: '1.0s' }, // the boundary itself switches to seconds + { ms: 1234, want: '1.2s' }, + { ms: 12_345, want: '12.3s' }, + ])('formatLatencyMs($ms) = $want', ({ ms, want }) => { + expect(formatLatencyMs(ms)).toBe(want) + }) +}) diff --git a/webui/src/lib/format.ts b/webui/src/lib/format.ts index c0b332b..00426f3 100644 --- a/webui/src/lib/format.ts +++ b/webui/src/lib/format.ts @@ -147,6 +147,21 @@ export function formatModelCostHover(inputUsd: number, outputUsd: number): strin return `in ${formatUsdPerMTok(inputUsd)} / out ${formatUsdPerMTok(outputUsd)} per MTok` } +/** + * formatLatencyMs renders a millisecond average (admin.go's + * adminLatencyView.avgTtfbMs/avgDurationMs — feat: latency panel) as a + * compact duration: plain milliseconds below 1000 ("120ms"), one-decimal + * seconds at or above it ("1.2s"). These values are already averages + * (sum/count from the in-process latency accumulator), so rounding here + * carries none of formatCompactCount's boundary-crossing honesty concern + * above — there is no discrete threshold a caller could misread as + * crossed, just a plain rounded reading. + */ +export function formatLatencyMs(ms: number): string { + if (ms < 1000) return `${Math.round(ms)}ms` + return `${(ms / 1000).toFixed(1)}s` +} + /** formatAgo renders an ISO timestamp as "(Ns ago)", or "" for the unset zero-time sentinel. */ export function formatAgo(iso: string | undefined): string { if (!iso || iso === ZERO_TIME) return '' diff --git a/webui/src/types/api.ts b/webui/src/types/api.ts index 11924e4..b4a0478 100644 --- a/webui/src/types/api.ts +++ b/webui/src/types/api.ts @@ -40,6 +40,24 @@ export interface AdminModelMetaView { outputPerMTokUsd?: number } +/** + * One provider's one stream-state's compact latency summary (admin.go: + * adminLatencyView — feat: instrument upstream latency), sourced from the + * SAME in-process g.latency accumulator /metrics reads, never Redis — + * PER-REPLICA, not fleet-wide (see AdminProviderView.latency's own doc + * comment). Every field is optional/undefined, not a plain 0: each is + * `omitempty` on the Go side, so undefined means "no observations for + * this field", never a fabricated zero. avgTtfbMs in particular must + * never be treated as interchangeable with avgDurationMs — see + * AdminProviderView.latency's doc comment for why they diverge for + * non-streaming traffic. + */ +export interface AdminLatencyView { + avgTtfbMs?: number + avgDurationMs?: number + count?: number +} + export interface AdminProviderView { name: string type: string @@ -96,6 +114,34 @@ export interface AdminProviderView { failuresMinute: number /** Per-model breakdown of the counters above, keyed by upstream model id — always present (possibly empty), never omitted. */ modelRates: Record + /** + * latency is this provider's compact per-stream average TTFB/duration + * summary (admin.go: adminProviderView.Latency — feat: instrument + * upstream latency), keyed by "streaming"/"non-streaming" — the same + * two string keys buildAdminLatencyViews (admin.go) emits, mirrored + * here as a literal union rather than a bare Record so a + * caller cannot accidentally index a third, nonexistent stream state. + * Undefined entirely for a provider with no observations yet, or a + * deployment with metrics collection gated off — undefined, never an + * empty object, so "no data" is never confused with "data says zero". + * + * PER-REPLICA, IN-PROCESS: sourced from the same in-process + * accumulator the rate-limit rejection counters read, never Redis. + * This deployment runs multiple Traefik replicas, so any single poll + * reflects only whichever replica answered it, and a freshly restarted + * pod shows a short observation window — never render this as if it + * were a fleet-wide average. + * + * STREAMING VS NON-STREAMING ARE NOT COMPARABLE: avgTtfbMs is the + * load-sensitive figure, and only for stream === 'streaming' — a + * non-streaming provider buffers its whole completion before sending + * anything, so its own avgTtfbMs is approximately equal to its own + * avgDurationMs and carries the identical output-length contamination + * (a long completion legitimately takes longer than a short one on an + * equally healthy provider). Never average the two stream states + * together or present either one as a bare, unlabeled "latency". + */ + latency?: Partial> } export interface AdminRedisView {