test(admin): probe the new admin routes under Yaegi and across replicas

yaegi-check now calls the new routes under the interpreter. It checks the overview replica and warnings, a 429 followed by a rate_limit event, the kind and status of upstream and timeout events, span=24 on the models ranking, and rejectionsPerDay in usage. A new integration test with two replicas triggers a 429 on one replica and reads the event from the other through Redis. It checks that the two replicas report different ids and the same total rejectionsPerDay. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
2026-10-13 04:58:50 +00:00 · 2026-09-23 13:09:53 +01:00
parent 6776981675
commit 25e6a7779e
2 changed files with 413 additions and 2 deletions
@@ -564,6 +564,112 @@ func TestCrossReplicaLimits(t *testing.T) {
 	}
 }

+// TestCrossReplicaEventsAndOverview covers plan section 5's own
+// "two-replica" integration bullet (v0.3 dashboard task, F3/F4/F9): a
+// rate-limit rejection recorded on replica 1 must be readable through
+// replica 2's own GET /admin/api/events (source "redis" — proving the
+// shared Redis-backed mirror, not merely replica 1's own local ring,
+// answers replica 2's read), attributed to replica 1's own replica id;
+// each replica's own GET /admin/api/overview reports a DIFFERENT replica
+// id (Q9: os.Hostname() -> $HOSTNAME -> "unknown" — two distinct
+// containers); and the fleet-wide total.rejectionsPerDay counter (F4, Q6:
+// "attribute rejections to total/all too") reads identically through
+// either replica, since both read the SAME shared Redis counter, not a
+// per-process one.
+func TestCrossReplicaEventsAndOverview(t *testing.T) {
+	flushRedis(t)
+
+	overview1Resp, overview1Body := doJSON(t, http.MethodGet, traefik1URL+"/admin/api/overview", adminKey, nil)
+	if overview1Resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s/admin/api/overview: status = %d, body=%#v", traefik1URL, overview1Resp.StatusCode, overview1Body)
+	}
+	replica1, _ := overview1Body["replica"].(string)
+	if replica1 == "" {
+		t.Fatalf("GET %s/admin/api/overview: replica = %q, want non-empty", traefik1URL, replica1)
+	}
+
+	overview2Resp, overview2Body := doJSON(t, http.MethodGet, traefik2URL+"/admin/api/overview", adminKey, nil)
+	if overview2Resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s/admin/api/overview: status = %d, body=%#v", traefik2URL, overview2Resp.StatusCode, overview2Body)
+	}
+	replica2, _ := overview2Body["replica"].(string)
+	if replica2 == "" {
+		t.Fatalf("GET %s/admin/api/overview: replica = %q, want non-empty", traefik2URL, replica2)
+	}
+	if replica1 == replica2 {
+		t.Fatalf("GET /admin/api/overview: replica1 = %q, replica2 = %q, want two distinct replica ids (two separate containers)", replica1, replica2)
+	}
+
+	// Exhaust bob's shared requestsPerMinute:3 bucket, all four requests
+	// against replica 1 ONLY (unlike TestCrossReplicaLimits, above, which
+	// deliberately alternates), so the resulting rate_limit event is
+	// unambiguously attributed to replica1's own id.
+	reqBody := map[string]any{
+		"model":    "openai/gpt-mock",
+		"messages": []map[string]any{{"role": "user", "content": "hi"}},
+	}
+	var saw429 bool
+	for i := 0; i < 4; i++ {
+		resp, body := doJSON(t, http.MethodPost, traefik1URL+"/v1/chat/completions", bobKey, reqBody)
+		switch resp.StatusCode {
+		case http.StatusOK:
+		case http.StatusTooManyRequests:
+			saw429 = true
+		default:
+			t.Fatalf("unexpected status %d for bob's request %d against replica 1, body=%#v", resp.StatusCode, i, body)
+		}
+	}
+	if !saw429 {
+		t.Fatal("want at least one 429 among 4 requests against bob's requestsPerMinute:3 bucket on replica 1")
+	}
+
+	// Read GET /admin/api/events THROUGH REPLICA 2 — proving the shared
+	// Redis mirror, not just replica 1's own local ring, carries the
+	// event.
+	eventsResp, eventsBody := doJSON(t, http.MethodGet, traefik2URL+"/admin/api/events?limit=10", adminKey, nil)
+	if eventsResp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s/admin/api/events: status = %d, body=%#v", traefik2URL, eventsResp.StatusCode, eventsBody)
+	}
+	if source, _ := eventsBody["source"].(string); source != "redis" {
+		t.Fatalf(`GET %s/admin/api/events: source = %q, want "redis" (replica 2 has no local record of an event replica 1 recorded)`, traefik2URL, source)
+	}
+	events, _ := eventsBody["events"].([]any)
+	if len(events) == 0 {
+		t.Fatalf("GET %s/admin/api/events: want at least one event, body=%#v", traefik2URL, eventsBody)
+	}
+	first, ok := events[0].(map[string]any)
+	if !ok {
+		t.Fatalf("GET %s/admin/api/events: events[0] is not an object: %v", traefik2URL, events[0])
+	}
+	if kind, _ := first["kind"].(string); kind != "rate_limit" {
+		t.Fatalf(`GET %s/admin/api/events: events[0] (newest first).kind = %q, want "rate_limit"`, traefik2URL, kind)
+	}
+	if eventReplica, _ := first["replica"].(string); eventReplica != replica1 {
+		t.Fatalf("GET %s/admin/api/events: events[0].replica = %q, want %q (replica 1, where the rejection actually happened)", traefik2URL, eventReplica, replica1)
+	}
+
+	// total.rejectionsPerDay (F4) must read identically through either
+	// replica.
+	usage1Resp, usage1Body := doJSON(t, http.MethodGet, traefik1URL+"/admin/api/usage", adminKey, nil)
+	if usage1Resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s/admin/api/usage: status = %d, body=%#v", traefik1URL, usage1Resp.StatusCode, usage1Body)
+	}
+	usage2Resp, usage2Body := doJSON(t, http.MethodGet, traefik2URL+"/admin/api/usage", adminKey, nil)
+	if usage2Resp.StatusCode != http.StatusOK {
+		t.Fatalf("GET %s/admin/api/usage: status = %d, body=%#v", traefik2URL, usage2Resp.StatusCode, usage2Body)
+	}
+	total1, _ := usage1Body["total"].(map[string]any)
+	total2, _ := usage2Body["total"].(map[string]any)
+	rej1, _ := total1["rejectionsPerDay"].(float64)
+	rej2, _ := total2["rejectionsPerDay"].(float64)
+	if rej1 < 1 {
+		t.Fatalf("GET %s/admin/api/usage: total.rejectionsPerDay = %v, want >= 1 after a real 429", traefik1URL, total1["rejectionsPerDay"])
+	}
+	if rej1 != rej2 {
+		t.Fatalf("total.rejectionsPerDay differs across replicas: replica1 = %v, replica2 = %v, want identical (same shared Redis counter)", rej1, rej2)
+	}
+}
+
 // TestUsersFileHotReload covers integration test 5: appending a new user
 // to the file-sourced users.json (bind-mounted read-only into both
 // Traefik containers, edited here on the host) makes their key valid
@@ -381,6 +381,19 @@ func run() error {
 	}))
 	defer provenanceEstUpstream.Close()

+	// eventsUpstreamErrorUpstream always 500s — see
+	// eventsUpstreamErrorProviderName's own doc comment for why this
+	// dedicated, single-candidate provider exists rather than reusing
+	// failoverAUpstream (which a correctly working failover loop never
+	// surfaces to the client, so its own 500 never reaches
+	// handleAdapterErrorEnvelope's client-facing write).
+	eventsUpstreamErrorUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write([]byte(`{"error":{"message":"eventsprobe-upstream always fails","type":"server_error"}}`))
+	}))
+	defer eventsUpstreamErrorUpstream.Close()
+
 	// mcpProbeUpstream answers federation's outbound tools/call with a
 	// body deliberately larger than mcpBackendCallResponseMaxBytes, so
 	// exerciseHandler's POST /mcp probe below drives doBackendJSONRPC's
@@ -674,6 +687,10 @@ func run() error {
 		// provenanceEstProviderName (feat: expose token-accounting
 		// provenance) — see its own const block doc comment above.
 		`"` + provenanceEstProviderName + `":{"type":"openai","baseUrl":"` + provenanceEstUpstream.URL + `","apiKey":"sk-up","models":["` + provenanceEstProviderModel + `"]},` +
+		// eventsUpstreamErrorProviderName (F3 hook 2, coordinator request)
+		// — see its own const block doc comment above for why this
+		// provider carries no failover candidate.
+		`"` + eventsUpstreamErrorProviderName + `":{"type":"openai","baseUrl":"` + eventsUpstreamErrorUpstream.URL + `","apiKey":"sk-up","models":["` + eventsUpstreamErrorProviderModel + `"]},` +
 		// "anthropic" backs the /v1/messages passthrough probes
 		// (exerciseMessagesRoute, below): a real anthropic-type provider,
 		// interpreted end to end through
@@ -797,7 +814,12 @@ func run() error {
 		// (primary, unrestricted) and round3P2FriendsGroupName
 		// (paths-restricted), plus a personal grant restricting
 		// "round3-shared" to round3PersonalModelID alone.
-		`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true},{"name":` + string(trickyNameJSON) + `,"group":"default","apiKey":"sk-tricky"},{"name":"bare-winner-friend","group":"` + bareWinnerGroupName + `","apiKey":"` + bareWinnerFriendAPIKey + `"},{"name":"multi-group-user","group":"` + multiGroupEngGroupName + `","groups":["` + multiGroupOpsGroupName + `"],"apiKey":"` + multiGroupUserAPIKey + `","providers":["multi-group-b"],"models":["` + multiGroupAllowedModelID + `"]},{"name":"round3-p1-user","group":"` + round3P1AGroupName + `","groups":["` + round3P1BGroupName + `"],"apiKey":"` + round3P1UserAPIKey + `"},{"name":"round3-p2-user","group":"` + round3P2HomeGroupName + `","groups":["` + round3P2FriendsGroupName + `"],"apiKey":"` + round3P2UserAPIKey + `","providers":["round3-shared"],"models":["` + round3PersonalModelID + `"]}]}}`
+		`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true},{"name":` + string(trickyNameJSON) + `,"group":"default","apiKey":"sk-tricky"},{"name":"bare-winner-friend","group":"` + bareWinnerGroupName + `","apiKey":"` + bareWinnerFriendAPIKey + `"},{"name":"multi-group-user","group":"` + multiGroupEngGroupName + `","groups":["` + multiGroupOpsGroupName + `"],"apiKey":"` + multiGroupUserAPIKey + `","providers":["multi-group-b"],"models":["` + multiGroupAllowedModelID + `"]},{"name":"round3-p1-user","group":"` + round3P1AGroupName + `","groups":["` + round3P1BGroupName + `"],"apiKey":"` + round3P1UserAPIKey + `"},{"name":"round3-p2-user","group":"` + round3P2HomeGroupName + `","groups":["` + round3P2FriendsGroupName + `"],"apiKey":"` + round3P2UserAPIKey + `","providers":["round3-shared"],"models":["` + round3PersonalModelID + `"]},` +
+		// The eighth inline user (eventsProbeUserAPIKey) is
+		// exerciseRateLimitEventAndRejections' own — group "default"
+		// (allow-all), but with its own requestsPerMinute:1 limit so a
+		// second call under this key alone 429s deterministically.
+		`{"name":"events-probe","group":"default","apiKey":"` + eventsProbeUserAPIKey + `","limits":{"requestsPerMinute":1}}]}}`
 	if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
 		return fmt.Errorf("decode attempt-accounting harness override into the interpreted Config: %w", err)
 	}
@@ -935,7 +957,31 @@ func run() error {
 	// chat completion of its own, and asserts on a kindModel counter no
 	// earlier probe touches, so it can neither perturb nor be perturbed by
 	// the provider-level counters they assert on.
-	return exerciseModelUsageRanking(handler)
+	if err := exerciseModelUsageRanking(handler); err != nil {
+		return err
+	}
+	// exerciseAdminOverviewReplicaAndWarnings/exerciseUsageModelsSpan (F4/
+	// F9/F10/F2, v0.3 dashboard task) run next: both only READ admin
+	// endpoints, so neither perturbs nor is perturbed by anything above.
+	if err := exerciseAdminOverviewReplicaAndWarnings(handler); err != nil {
+		return err
+	}
+	if err := exerciseUsageModelsSpan(handler); err != nil {
+		return err
+	}
+	// exerciseRateLimitEventAndRejections (F3/F4, v0.3 dashboard task)
+	// runs next, against its own dedicated requestsPerMinute:1 user
+	// (eventsProbeUserAPIKey), so its two calls cannot perturb any earlier
+	// probe's own per-user counters.
+	if err := exerciseRateLimitEventAndRejections(handler); err != nil {
+		return err
+	}
+	// exerciseUpstreamAndTimeoutEvents (F3 hook 2, coordinator request
+	// 2026-09-23) runs LAST: each of its two assertions reads GET
+	// /admin/api/events?limit=1 for the single newest entry right after
+	// producing it, so nothing after it depends on event ordering staying
+	// undisturbed.
+	return exerciseUpstreamAndTimeoutEvents(handler)
 }

 // builtinLookupModelID is a real, stable entry in the generated
@@ -1070,6 +1116,26 @@ const (
 	round3PersonalModelID    = "round3-shared/m1"
 )

+// eventsProbeUserAPIKey authenticates exerciseRateLimitEventAndRejections'
+// own inline user (run()'s "users" override): group "default" (allow-all,
+// like every other probe user), but with its own requestsPerMinute:1
+// limit, so a SECOND call under this key alone 429s regardless of what
+// every other probe already did to testDataUserAPIKey's own counters —
+// no other probe may reuse this key.
+const eventsProbeUserAPIKey = "sk-events-probe" // #nosec G101 -- test fixture literal, not a real credential
+
+// eventsUpstreamErrorProviderName/-Model (F3 hook 2, coordinator request
+// 2026-09-23) back exerciseUpstreamAndTimeoutEvents: a SINGLE provider —
+// deliberately no failover candidate configured for this model id — whose
+// upstream always 500s, so the client-facing response IS the classified
+// *providerHTTPError handleAdapterErrorEnvelope hands to writeUpstream,
+// and recordUpstreamEvent's default branch runs on every call, not just
+// the first of a failover attempt sequence.
+const (
+	eventsUpstreamErrorProviderName  = "eventsprobe-upstream"
+	eventsUpstreamErrorProviderModel = "gpt-eventsprobe-upstream-test"
+)
+
 // mcpProbeServerName is the federated MCP server run() configures against
 // mcpProbeUpstream, and mcpProbeToolName is a tool id carrying its
 // "<server>_" prefix so resolveFederatedTool routes POST /mcp's
@@ -3378,3 +3444,242 @@ func pollModelRanking(handler http.Handler) ([]any, error) {
 		time.Sleep(10 * time.Millisecond)
 	}
 }
+
+// exerciseAdminOverviewReplicaAndWarnings proves GET /admin/api/overview's
+// two new fields (F4/F9 replica id, F10 config-warnings collection, v0.3
+// dashboard task) are populated correctly under the REAL interpreter:
+// replica must be non-empty (g.replica: os.Hostname() -> $HOSTNAME ->
+// "unknown", llmgateway.go) and warnings must decode as a genuine JSON
+// array, never null (configWarningsSnapshot's own "never nil" contract,
+// logger.go) — a null here would mean noteConfigWarning's slice append
+// under the interpreter produced a nil rather than an empty slice, a
+// construct class this codebase has seen diverge before. Content is
+// fixture-dependent (this harness's own config may or may not trip a
+// warnf during construction), so only presence/type is asserted, not any
+// particular message.
+func exerciseAdminOverviewReplicaAndWarnings(handler http.Handler) error {
+	req := httptest.NewRequest(http.MethodGet, "/admin/api/overview", nil)
+	req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		return fmt.Errorf("GET /admin/api/overview (replica/warnings probe): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
+	}
+	var body map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+		return fmt.Errorf("decode GET /admin/api/overview body: %w", err)
+	}
+	replica, _ := body["replica"].(string)
+	if replica == "" {
+		return fmt.Errorf(`GET /admin/api/overview: "replica" = %q, want non-empty (F4/F9: os.Hostname() -> $HOSTNAME -> "unknown")`, replica)
+	}
+	if _, ok := body["warnings"].([]any); !ok {
+		return fmt.Errorf(`GET /admin/api/overview: "warnings" did not decode as a JSON array (want never-nil, F10): %v`, body["warnings"])
+	}
+	fmt.Println("yaegi-check: GET /admin/api/overview carries a non-empty replica id and a never-nil warnings array under the interpreter")
+	return nil
+}
+
+// exerciseUsageModelsSpan proves GET /admin/api/usage/models' optional
+// ?span= parameter (F2, v0.3 dashboard task) is parsed, validated, and
+// echoed back correctly under the REAL interpreter: modelSpanTotals'
+// per-id historyBucketKeys loop and chunkedModelSpanTotals' chunking
+// arithmetic (admin.go/limits.go) are both new, interpreter-untested
+// code this harness's own compiled test suite cannot vouch for. window
+// "hour" caps span at 48 (historyMaxSpan), so 24 is comfortably valid.
+func exerciseUsageModelsSpan(handler http.Handler) error {
+	req := httptest.NewRequest(http.MethodGet, "/admin/api/usage/models?metric=req&window=hour&span=24", nil)
+	req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		return fmt.Errorf("GET /admin/api/usage/models?span=24: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
+	}
+	var body map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+		return fmt.Errorf("decode GET /admin/api/usage/models body: %w", err)
+	}
+	if span, _ := body["span"].(float64); span != 24 {
+		return fmt.Errorf(`GET /admin/api/usage/models?span=24: "span" = %v, want 24 (F2's own echoed, resolved span)`, body["span"])
+	}
+	fmt.Println("yaegi-check: GET /admin/api/usage/models honors ?span=24 on window=hour and echoes it back under the interpreter")
+	return nil
+}
+
+// exerciseRateLimitEventAndRejections proves a real checkAndCount
+// rejection is BOTH recorded to GET /admin/api/events as a rate_limit
+// event (F3 hook 1) AND folded into GET /admin/api/usage's fleet-wide
+// total.rejectionsPerDay counter (F4, Q6: "attribute rejections to
+// total/all too") under the REAL interpreter — settleRejection's own
+// single storeIncrMulti round trip (limits.go) and recordLimitEvent's
+// event-ring append (events.go) are both new code no compiled test here
+// can vouch for. eventsProbeUserAPIKey's own dedicated
+// requestsPerMinute:1 limit (run()'s "users" override) makes the second
+// call 429 deterministically, regardless of what every other probe
+// already did to testDataUserAPIKey's own counters.
+func exerciseRateLimitEventAndRejections(handler http.Handler) error {
+	body := `{"model":"` + testDataWantModel + `","messages":[{"role":"user","content":"hi"}]}`
+
+	req1 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
+	req1.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey)
+	req1.Header.Set("Content-Type", "application/json")
+	rec1 := httptest.NewRecorder()
+	handler.ServeHTTP(rec1, req1)
+	if rec1.Code != http.StatusOK {
+		return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 1st call): status = %d, want 200, body=%s", rec1.Code, rec1.Body.String())
+	}
+
+	req2 := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(body))
+	req2.Header.Set("Authorization", "Bearer "+eventsProbeUserAPIKey)
+	req2.Header.Set("Content-Type", "application/json")
+	rec2 := httptest.NewRecorder()
+	handler.ServeHTTP(rec2, req2)
+	if rec2.Code != http.StatusTooManyRequests {
+		return fmt.Errorf("POST /v1/chat/completions (rate-limit-event harness, 2nd call): status = %d, want 429 (over requestsPerMinute:1), body=%s", rec2.Code, rec2.Body.String())
+	}
+
+	eventsReq := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=10", nil)
+	eventsReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
+	eventsRec := httptest.NewRecorder()
+	handler.ServeHTTP(eventsRec, eventsReq)
+	if eventsRec.Code != http.StatusOK {
+		return fmt.Errorf("GET /admin/api/events?limit=10: status = %d, want 200, body=%s", eventsRec.Code, eventsRec.Body.String())
+	}
+	var eventsBody map[string]any
+	if err := json.Unmarshal(eventsRec.Body.Bytes(), &eventsBody); err != nil {
+		return fmt.Errorf("decode GET /admin/api/events body: %w", err)
+	}
+	events, _ := eventsBody["events"].([]any)
+	if len(events) == 0 {
+		return fmt.Errorf("GET /admin/api/events: want at least one event, body=%s", eventsRec.Body.String())
+	}
+	first, ok := events[0].(map[string]any)
+	if !ok {
+		return fmt.Errorf("GET /admin/api/events: events[0] is not an object: %v", events[0])
+	}
+	if kind, _ := first["kind"].(string); kind != "rate_limit" {
+		return fmt.Errorf(`GET /admin/api/events: events[0] (newest first).kind = %q, want "rate_limit"`, kind)
+	}
+	if route, _ := first["route"].(string); route != "chat/completions" {
+		return fmt.Errorf(`GET /admin/api/events: events[0].route = %q, want "chat/completions"`, route)
+	}
+
+	usageReq := httptest.NewRequest(http.MethodGet, "/admin/api/usage", nil)
+	usageReq.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
+	usageRec := httptest.NewRecorder()
+	handler.ServeHTTP(usageRec, usageReq)
+	if usageRec.Code != http.StatusOK {
+		return fmt.Errorf("GET /admin/api/usage: status = %d, want 200, body=%s", usageRec.Code, usageRec.Body.String())
+	}
+	var usageBody map[string]any
+	if err := json.Unmarshal(usageRec.Body.Bytes(), &usageBody); err != nil {
+		return fmt.Errorf("decode GET /admin/api/usage body: %w", err)
+	}
+	total, ok := usageBody["total"].(map[string]any)
+	if !ok {
+		return fmt.Errorf("GET /admin/api/usage: no total object: %s", usageRec.Body.String())
+	}
+	if rejPerDay, _ := total["rejectionsPerDay"].(float64); rejPerDay < 1 {
+		return fmt.Errorf("GET /admin/api/usage: total.rejectionsPerDay = %v, want >= 1 after a real 429 (F4)", total["rejectionsPerDay"])
+	}
+
+	fmt.Println("yaegi-check: a real 429 recorded a rate_limit event (GET /admin/api/events) and incremented total.rejectionsPerDay (GET /admin/api/usage) under the interpreter")
+	return nil
+}
+
+// exerciseUpstreamAndTimeoutEvents proves recordUpstreamEvent's own
+// classification (events.go, F3 hook 2) runs correctly under the REAL
+// interpreter. Added per a backend verifier's finding (coordinator
+// request, 2026-09-23): Yaegi v0.16.1 was found to leave kind/status
+// empty for a multi-value assignment whose right-hand side contains a
+// method call — exactly recordUpstreamEvent's own
+// `kind, status, message = eventKindTimeout, http.StatusGatewayTimeout,
+// err.Error()` shape — a construct class this codebase has repeatedly
+// seen diverge under the interpreter (this package's own doc comment).
+//
+// eventsUpstreamErrorProviderName is a single, no-failover-candidate
+// provider that always 500s, so the client-facing response IS the
+// classified *providerHTTPError and the newest event must be kind
+// "upstream" with that same 500 status. timeoutProviderName
+// (exerciseRequestTimeout's own fixture, reused here rather than adding
+// a third upstream — cheap) then makes the newest event a "timeout" one:
+// its client-facing response is 502 (exerciseRequestTimeout's own
+// assertion), but the EVENT's own Status field is independently set to
+// http.StatusGatewayTimeout (504) by recordUpstreamEvent — the two are
+// deliberately different fields of different origin, both asserted here.
+func exerciseUpstreamAndTimeoutEvents(handler http.Handler) error {
+	upstreamReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
+		`{"model":"`+eventsUpstreamErrorProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
+	))
+	upstreamReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
+	upstreamReq.Header.Set("Content-Type", "application/json")
+	upstreamRec := httptest.NewRecorder()
+	handler.ServeHTTP(upstreamRec, upstreamReq)
+	if upstreamRec.Code != http.StatusInternalServerError {
+		return fmt.Errorf("POST /v1/chat/completions (upstream-event harness, provider %q): status = %d, want 500 (no failover candidate configured), body=%s", eventsUpstreamErrorProviderName, upstreamRec.Code, upstreamRec.Body.String())
+	}
+	kind, status, err := latestEventKindAndStatus(handler)
+	if err != nil {
+		return err
+	}
+	if kind != "upstream" {
+		return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "upstream" after a real 500`, kind)
+	}
+	if status != http.StatusInternalServerError {
+		return fmt.Errorf("GET /admin/api/events: newest (upstream) event status = %d, want 500", status)
+	}
+
+	timeoutReq := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(
+		`{"model":"`+timeoutProviderName+`/`+timeoutProviderModel+`","messages":[{"role":"user","content":"hi"}]}`,
+	))
+	timeoutReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
+	timeoutReq.Header.Set("Content-Type", "application/json")
+	timeoutRec := httptest.NewRecorder()
+	handler.ServeHTTP(timeoutRec, timeoutReq)
+	if timeoutRec.Code != http.StatusBadGateway {
+		return fmt.Errorf("POST /v1/chat/completions (timeout-event harness, provider %q): status = %d, want 502 (upstream connection error, provider-attributed — exerciseRequestTimeout's own contract), body=%s", timeoutProviderName, timeoutRec.Code, timeoutRec.Body.String())
+	}
+	kind, status, err = latestEventKindAndStatus(handler)
+	if err != nil {
+		return err
+	}
+	if kind != "timeout" {
+		return fmt.Errorf(`GET /admin/api/events: newest event kind = %q, want "timeout" after a real request-timeout abort`, kind)
+	}
+	if status != http.StatusGatewayTimeout {
+		return fmt.Errorf("GET /admin/api/events: newest (timeout) event status = %d, want 504 (recordUpstreamEvent's own fixed value, distinct from the 502 the client saw)", status)
+	}
+
+	fmt.Println("yaegi-check: recordUpstreamEvent classified a real 500 as kind \"upstream\"/500 and a real request-timeout abort as kind \"timeout\"/504 under the interpreter")
+	return nil
+}
+
+// latestEventKindAndStatus reads GET /admin/api/events?limit=1 as admin
+// and returns the single newest event's kind/status — the shared read
+// exerciseUpstreamAndTimeoutEvents' two assertions both drive, limit=1
+// so each call cares only about the entry its own immediately preceding
+// request produced, never an earlier probe's.
+func latestEventKindAndStatus(handler http.Handler) (kind string, status int, err error) {
+	req := httptest.NewRequest(http.MethodGet, "/admin/api/events?limit=1", nil)
+	req.Header.Set("Authorization", "Bearer "+attemptAccountingAdminAPIKey)
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: status = %d, want 200, body=%s", rec.Code, rec.Body.String())
+	}
+	var body map[string]any
+	if jsonErr := json.Unmarshal(rec.Body.Bytes(), &body); jsonErr != nil {
+		return "", 0, fmt.Errorf("decode GET /admin/api/events body: %w", jsonErr)
+	}
+	events, _ := body["events"].([]any)
+	if len(events) == 0 {
+		return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: want at least one event, body=%s", rec.Body.String())
+	}
+	first, ok := events[0].(map[string]any)
+	if !ok {
+		return "", 0, fmt.Errorf("GET /admin/api/events?limit=1: events[0] is not an object: %v", events[0])
+	}
+	k, _ := first["kind"].(string)
+	s, _ := first["status"].(float64)
+	return k, int(s), nil
+}