mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-16 05:37:28 +00:00
merge: serve the Anthropic Messages API inbound
# Conflicts: # tools/yaegi-check/main.go
This commit is contained in:
@@ -28,6 +28,10 @@ combination.
|
||||
and from Anthropic's and Google Gemini's own wire formats, so any
|
||||
OpenAI-SDK-compatible client can talk to all three providers through one
|
||||
API shape.
|
||||
- An Anthropic-compatible surface (`POST /v1/messages`) for Anthropic SDK
|
||||
clients — including Claude Code — to talk to any configured provider in
|
||||
Anthropic's own Messages API shape; see [Anthropic Messages API
|
||||
(`/v1/messages`)](#anthropic-messages-api-v1messages).
|
||||
- Native passthrough per provider (`/{provider}/...`) for callers that want
|
||||
a provider's own wire format untouched, with the gateway injecting that
|
||||
provider's own API key and still enforcing auth and limits.
|
||||
@@ -998,6 +1002,99 @@ only once it has something to authorize:
|
||||
will see every Gemini passthrough request denied. Route Gemini traffic
|
||||
for a model-restricted group through the unified `/v1/*` API instead.
|
||||
|
||||
## Anthropic Messages API (`/v1/messages`)
|
||||
|
||||
`POST /v1/messages` serves Anthropic SDK clients — including Claude
|
||||
Code — directly, in Anthropic's own Messages API request/response shape:
|
||||
the inbound counterpart to the outbound-only Anthropic support the
|
||||
unified routes already had. From model resolution through response
|
||||
caching, usage accounting, and adapter-error handling, this route shares
|
||||
one implementation with `/v1/chat/completions`
|
||||
(`runMeteredCall`, `routes_unified.go`) — the same group authorization
|
||||
(see [`GroupConfig`](#groupconfig)), [limits and
|
||||
accounting](#limits-and-accounting), and [caching](#caching) semantics
|
||||
apply. Only the wire shape and the client-facing error envelope are
|
||||
specific to this route.
|
||||
|
||||
### Authentication
|
||||
|
||||
Anthropic SDKs authenticate with the `x-api-key` header, not
|
||||
`Authorization: Bearer`. This route accepts **both** — the identical
|
||||
credential lookup (`authStore.identify`, `auth.go`) every other route
|
||||
uses already reads either header, `Authorization: Bearer` taking
|
||||
priority if a request somehow carries both — so no separate credential
|
||||
path exists for this route. A failed auth answers in Anthropic's own
|
||||
error shape (`{"type":"error","error":{"type":"authentication_error",
|
||||
"message":"..."}}`), never the OpenAI-shaped
|
||||
`{"error":{"type":...,"code":...}}` every other route uses.
|
||||
|
||||
### Translation vs. passthrough
|
||||
|
||||
Behavior depends entirely on the resolved provider's configured `type`:
|
||||
|
||||
| Resolved provider `type` | Behavior |
|
||||
|---|---|
|
||||
| `anthropic` | **Passthrough.** The request reaches that provider's own `/v1/messages` endpoint untranslated — no message-body translation of any kind. The response reaches the client BYTE-FOR-BYTE identical, with one exception: `"model"` is rewritten to the client's own requested alias, the same routing rewrite every route applies before any adapter call — not a translation of the response body. This is a real byte-level guarantee, not an approximation: every other field's value is copied from the upstream's raw bytes without ever passing through a Go value, so a JSON integer beyond `float64`'s exact range (inside, say, a `tool_use` block's arbitrary `input`) survives intact instead of being silently rounded. |
|
||||
| `openai` or `gemini` | **Full bidirectional translation**, through the same `providerAdapter.chatCompletion` a `/v1/chat/completions` request against that provider already uses — its contract is OpenAI-shape in, OpenAI-shape out regardless of provider type. The Anthropic-shaped request is translated to OpenAI's shape before the call; that provider's OpenAI-shaped response is translated back to Anthropic's shape before it reaches the client. |
|
||||
|
||||
A few translation specifics worth knowing:
|
||||
|
||||
- **`anthropic-beta` is forwarded** to an `anthropic`-type upstream (an
|
||||
explicit allowlist — `Authorization`, `x-api-key`, `Cookie`, and every
|
||||
hop-by-hop header are never forwarded, and the gateway's own credential
|
||||
injection always overrides whatever the allowlist copied), so Claude
|
||||
Code and the SDKs can still opt into 1M context, token-efficient
|
||||
tools, computer use, and other beta features through this route.
|
||||
- **Prompt-cache tokens are billed.** Anthropic's
|
||||
`cache_creation_input_tokens` and `cache_read_input_tokens` fold into
|
||||
the same combined prompt-token count every other route's budgets
|
||||
track, alongside `input_tokens` — not dropped, and not priced at a
|
||||
separate cache-tier rate.
|
||||
- **`tool_result.is_error`** has no native OpenAI tool-message field.
|
||||
`content` — a string, Anthropic's content-block array form (Claude
|
||||
Code sends this), or absent — is normalized to a plain string first
|
||||
(OpenAI's own tool message requires one; a `null` content is
|
||||
rejected), then gets an `"Error: "` prefix, so an OpenAI-backed
|
||||
agentic loop still sees the failure signal regardless of which shape
|
||||
the client sent.
|
||||
- **Reasoning models** (OpenAI's `o1`/`o3`/`o4`/`gpt-5` families) get
|
||||
`max_completion_tokens` instead of `max_tokens` — the field those
|
||||
models reject with a 400.
|
||||
- **`thinking`** (Anthropic's extended-thinking config) has no OpenAI
|
||||
equivalent and is dropped when translating to an `openai`/`gemini`
|
||||
target; the gateway logs a warning naming the dropped field rather
|
||||
than discarding it silently.
|
||||
|
||||
### Streaming is not yet supported
|
||||
|
||||
`"stream"` set to anything other than the JSON literal `false`, `null`,
|
||||
or an absent key gets a clean **400** `invalid_request_error` — never
|
||||
silently ignored, never answered non-streamed. `null` is treated the
|
||||
same as unset, so an SDK that serializes an unset optional field as
|
||||
explicit `null` never gets a false-positive rejection on an ordinary
|
||||
non-streaming call. This is deliberately a 400, not a 501: Anthropic's
|
||||
own SDKs retry any status ≥ 500, so a 501 here would turn every
|
||||
streaming call — the default for Claude Code — into a multi-request
|
||||
retry storm against the caller's own request-rate limit. The rejection
|
||||
happens *before* the rate limiter runs, so a streaming-rejected request
|
||||
never consumes budget either. A follow-up release adds the Anthropic SSE
|
||||
event translator (`message_start`/`content_block_delta`/`message_stop`)
|
||||
— a genuinely different event model from OpenAI's own stream shape this
|
||||
plugin already forwards on `/v1/chat/completions`.
|
||||
|
||||
### Route takeover
|
||||
|
||||
On a deployment running `passthroughUnknown: true`, `POST /v1/messages`
|
||||
used to fall through to `next` (the router's own backing service) before
|
||||
this route existed — nothing in this plugin recognized the path. Now the
|
||||
gateway intercepts it unconditionally: `next` is never reached for this
|
||||
path again, regardless of `passthroughUnknown`. This is intended — the
|
||||
whole point of this route is for the gateway itself to answer
|
||||
`/v1/messages` — but it is a real behavior change for any deployment
|
||||
currently proxying `/v1/messages` downstream through
|
||||
`passthroughUnknown`. Route that traffic elsewhere, or expect the
|
||||
gateway to answer it directly, before upgrading.
|
||||
|
||||
## Image and audio endpoints
|
||||
|
||||
`POST /v1/images/generations`, `POST /v1/audio/speech`, and
|
||||
@@ -1416,13 +1513,29 @@ the plugin's only non-standard-library runtime dependency.
|
||||
integration suite's true incremental-delivery timing assertion is gated
|
||||
behind `INTEGRATION_STREAMING=1` and skipped by default for exactly this
|
||||
reason — re-run it once the upstream issue is fixed.
|
||||
- **`POST /v1/messages` now intercepts unconditionally, even with
|
||||
`passthroughUnknown: true`.** Before this route existed, that path
|
||||
fell through to `next` on a `passthroughUnknown` deployment like any
|
||||
other unrecognized path. The gateway now answers it directly and
|
||||
`next` is never reached for it again — intended, but a real behavior
|
||||
change for a deployment currently proxying `/v1/messages` downstream.
|
||||
See [Anthropic Messages API — Route
|
||||
takeover](#route-takeover).
|
||||
- **A request rejected before it reaches a provider never consumes
|
||||
request-rate quota.** On the unified routes, a malformed body, a missing
|
||||
`model` field, or an unknown/denied model (404/403) is rejected before
|
||||
`limiter.checkAndCount` runs. On native passthrough and the MCP/A2A
|
||||
proxy, denied provider/target access (403), an unsupported `Upgrade`
|
||||
request (501), and a path-traversal attempt (400) are all checked before
|
||||
the limiter too. Once a request *does* reach a provider adapter,
|
||||
`limiter.checkAndCount` runs. `/v1/messages` goes further for one
|
||||
specific field: it decodes the body and rejects a streaming request
|
||||
(400) *before* the rate limiter runs too, the opposite relative order
|
||||
from the unified routes' own admission-before-decode default — a
|
||||
deliberate, route-scoped exception (see [Anthropic Messages
|
||||
API](#anthropic-messages-api-v1messages)), since Anthropic SDKs retry
|
||||
any 5xx and would otherwise burn multiple requests of budget per
|
||||
logical streaming call against a route that does not support streaming
|
||||
yet. On native passthrough and the MCP/A2A proxy, denied provider/target
|
||||
access (403), an unsupported `Upgrade` request (501), and a
|
||||
path-traversal attempt (400) are all checked before the limiter too.
|
||||
Once a request *does* reach a provider adapter,
|
||||
though, a translate-time 400 (an unsupported parameter combination) or
|
||||
Anthropic's embeddings 501 still count toward the caller's request-rate
|
||||
limit — the limiter already ran by then — though neither ever bills a
|
||||
|
||||
@@ -831,6 +831,25 @@ func (g *Gateway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodPost && r.URL.Path == messagesPath {
|
||||
// Anthropic SDKs authenticate with x-api-key, not Authorization:
|
||||
// Bearer — g.auth.identify (auth.go) already reads both headers,
|
||||
// Bearer taking priority when a caller somehow sends both, so no
|
||||
// route-specific auth handling is needed here beyond using the
|
||||
// same identify/logAuthEvent call every other route already
|
||||
// makes. A failed auth gets the Anthropic error shape
|
||||
// (writeAnthropicError, routes_messages.go), not writeOAIError's
|
||||
// — the one thing that differs from every route above.
|
||||
u, grp, ok := g.auth.identify(r)
|
||||
g.logAuthEvent(ok, authEventUserName(u), r)
|
||||
if !ok {
|
||||
writeAnthropicError(sw, http.StatusUnauthorized, "authentication_error", "invalid or missing API key")
|
||||
return
|
||||
}
|
||||
g.handleMessages(sw, r, u, grp)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Method == http.MethodPost && (r.URL.Path == imagesGenerationsPath || r.URL.Path == audioSpeechPath || r.URL.Path == audioTranscriptionsPath) {
|
||||
u, grp, ok := g.auth.identify(r)
|
||||
g.logAuthEvent(ok, authEventUserName(u), r)
|
||||
|
||||
+5
-5
@@ -42,7 +42,7 @@ const maxMultipartModelFieldBytes = 4096
|
||||
// multipart body. A read failure or invalid/missing JSON is a 400,
|
||||
// already written to sw; ok reports whether the caller may proceed.
|
||||
func (g *Gateway) decodeMediaJSONRequest(sw *statusTrackingWriter, r *http.Request) (req map[string]any, model string, ok bool) {
|
||||
release, admitted := g.acquireBodyAdmission(sw)
|
||||
release, admitted := g.acquireBodyAdmission(sw, writeOAIError)
|
||||
defer release()
|
||||
if !admitted {
|
||||
return nil, "", false
|
||||
@@ -100,7 +100,7 @@ func (g *Gateway) resolveMediaModel(sw *statusTrackingWriter, grp *group, reques
|
||||
func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request, u *user, grp *group) {
|
||||
sw := &statusTrackingWriter{ResponseWriter: w}
|
||||
|
||||
if _, ok := g.admitRequest(sw, u, grp); !ok {
|
||||
if _, ok := g.admitRequest(sw, u, grp, writeOAIError); !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -137,7 +137,7 @@ func (g *Gateway) handleImagesGenerations(w http.ResponseWriter, r *http.Request
|
||||
func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *user, grp *group) {
|
||||
sw := &statusTrackingWriter{ResponseWriter: w}
|
||||
|
||||
if _, ok := g.admitRequest(sw, u, grp); !ok {
|
||||
if _, ok := g.admitRequest(sw, u, grp, writeOAIError); !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -197,7 +197,7 @@ func (g *Gateway) handleAudioSpeech(w http.ResponseWriter, r *http.Request, u *u
|
||||
func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Request, u *user, grp *group) {
|
||||
sw := &statusTrackingWriter{ResponseWriter: w}
|
||||
|
||||
if _, ok := g.admitRequest(sw, u, grp); !ok {
|
||||
if _, ok := g.admitRequest(sw, u, grp, writeOAIError); !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -293,7 +293,7 @@ func (g *Gateway) handleAudioTranscriptions(w http.ResponseWriter, r *http.Reque
|
||||
// acquireBodyAdmission already wrote its own 503 to sw); body/oversize/
|
||||
// err are readCapped's own results, valid only when admitted is true.
|
||||
func (g *Gateway) readAdmittedCapped(sw *statusTrackingWriter, r io.Reader, limit int64) (body []byte, oversize, admitted bool, err error) {
|
||||
release, admitted := g.acquireBodyAdmission(sw)
|
||||
release, admitted := g.acquireBodyAdmission(sw, writeOAIError)
|
||||
defer release()
|
||||
if !admitted {
|
||||
return nil, false, false, nil
|
||||
|
||||
@@ -0,0 +1,510 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// messagesPath is the inbound Anthropic Messages API route this gateway
|
||||
// serves: POST /v1/messages. Distinct from anthropicMessagesPath
|
||||
// (provider_anthropic.go), which is the SAME literal path string but
|
||||
// names where an anthropic-type adapter's OWN outbound call lands on its
|
||||
// configured upstream — two different constants for two different
|
||||
// directions, kept apart so a future change to one is never mistaken for
|
||||
// the other.
|
||||
const messagesPath = "/v1/messages"
|
||||
|
||||
// cacheEndpointMessages is cacheKey's endpoint argument (cache.go) for
|
||||
// this route, mirroring cacheEndpointChat/cacheEndpointEmbeddings
|
||||
// (routes_unified.go): keeps a /v1/messages request from ever colliding
|
||||
// into a /v1/chat/completions cache entry even when a request happens to
|
||||
// canonicalize to an identical provider/upstreamModel/body — the two
|
||||
// routes answer in different wire shapes (Anthropic vs OpenAI), so a
|
||||
// cache hit across them would hand a client the wrong envelope entirely.
|
||||
const cacheEndpointMessages = "messages"
|
||||
|
||||
// anthropicPassthroughForwardedHeaders is the ALLOWLIST of client request
|
||||
// headers callAnthropicMessagesPassthrough forwards to an anthropic-type
|
||||
// upstream (item 5 fix, 2026-08-22 review): Claude Code and the Anthropic
|
||||
// SDKs send anthropic-beta to opt into features this gateway must not
|
||||
// silently disable on a route named "passthrough" — 1M context,
|
||||
// token-efficient tools, computer use, among others. Deliberately an
|
||||
// allowlist, never a blanket forward: Authorization, x-api-key, Cookie,
|
||||
// and every hop-by-hop header must never reach an upstream this gateway
|
||||
// itself authenticates against on the caller's behalf. injectAuth is
|
||||
// always applied AFTER this copy (callAnthropicMessagesPassthrough,
|
||||
// below), so it always wins if a future entry here ever collided with
|
||||
// what injectAuth itself sets.
|
||||
var anthropicPassthroughForwardedHeaders = []string{"anthropic-beta"}
|
||||
|
||||
// handleMessages implements POST /v1/messages: the Anthropic Messages API
|
||||
// shape, inbound. From model resolution through response caching, usage
|
||||
// accounting, and adapter-error handling, this route shares ONE
|
||||
// implementation with routes_unified.go's runUnified — runMeteredCall,
|
||||
// routes_unified.go (item 7 fix, 2026-08-22 review) — rather than
|
||||
// duplicating that pipeline. Only two things are genuinely specific to
|
||||
// this route: which wire shape a request/response is in (Anthropic, not
|
||||
// OpenAI — see callAnthropicMessagesPassthrough/callTranslatedMessages
|
||||
// below) and which error envelope shape a failure gets written in
|
||||
// (Anthropic's {"type":"error","error":{...}}, not OpenAI's
|
||||
// {"error":{...}} — see writeAnthropicError and its callers, below).
|
||||
//
|
||||
// ORDERING (item 2 fix, 2026-08-22 review): body admission + decode run
|
||||
// BEFORE admitRequest (the rate-limit check) on THIS route — the
|
||||
// opposite order from runUnified's own finding 1a. anthropic-sdk-python
|
||||
// retries any status >= 500, and Claude Code streams on every call: the
|
||||
// previous ordering (admitRequest first, streaming rejected only after
|
||||
// decode) meant every real client hitting this not-yet-streaming-capable
|
||||
// route burned three requests of rate-limit budget per logical call —
|
||||
// verified repro: requestsPerMinute 2, three attempts -> 501, 501, 429,
|
||||
// day counter reads 3. Deciding "is this streaming" requires the decoded
|
||||
// body, so decode has to happen first; the trade-off finding 1a accepts
|
||||
// for runUnified (an adversarial caller paying nothing to retry a
|
||||
// malformed body) does not apply here, since the common case hitting
|
||||
// this ordering is not adversarial retries but every legitimate
|
||||
// streaming-capable client that exists today.
|
||||
func (g *Gateway) handleMessages(w http.ResponseWriter, r *http.Request, u *user, grp *group) {
|
||||
sw := &statusTrackingWriter{ResponseWriter: w}
|
||||
|
||||
body, req, ok := g.readAndDecodeUnifiedBody(sw, r, writeAnthropicError)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
requestedModel, _ := req["model"].(string)
|
||||
if requestedModel == "" {
|
||||
writeAnthropicError(sw, http.StatusBadRequest, "invalid_request_error", "model is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Streaming is rejected cleanly and explicitly here, before the
|
||||
// rate-limit check, model resolution, or any upstream call: this cut
|
||||
// of the route ships non-streaming only (brief scope) — a follow-up
|
||||
// branch adds the Anthropic SSE event translator (message_start/
|
||||
// content_block_delta/message_stop), a genuinely different event
|
||||
// model from OpenAI's stream shape this route already knows how to
|
||||
// forward. A caller that asked for streaming must see a clear, typed
|
||||
// error, never a silently-ignored flag answered non-streamed.
|
||||
//
|
||||
// 400, not 501 (item 2 fix, 2026-08-22 review): anthropic-sdk-python's
|
||||
// retry classifies purely on status code and retries anything >= 500
|
||||
// — a 501 here turned every streaming call into an SDK-driven retry
|
||||
// storm against this route's own rate limit (this function's own doc
|
||||
// comment above has the verified repro). 400 maps to the SDK's
|
||||
// non-retryable BadRequestError, and invalid_request_error is already
|
||||
// the correct error type for a 400.
|
||||
if isStreamingRequested(req) {
|
||||
writeAnthropicError(sw, http.StatusBadRequest, "invalid_request_error", "streaming is not yet supported on /v1/messages")
|
||||
return
|
||||
}
|
||||
|
||||
scopes, ok := g.admitRequest(sw, u, grp, writeAnthropicError)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
g.runMeteredCall(sw, r, scopes, body, req, requestedModel, grp, cacheEndpointMessages, "messages route", writeAnthropicError, writeAnthropicProviderUpstreamError,
|
||||
func(a providerAdapter, ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error) {
|
||||
if a.typeName() == providerTypeAnthropic {
|
||||
return g.callAnthropicMessagesPassthrough(ctx, w, r.Header, a, req, requestedModel)
|
||||
}
|
||||
return g.callTranslatedMessages(ctx, w, a, req, requestedModel)
|
||||
})
|
||||
}
|
||||
|
||||
// isStreamingRequested reports whether req's "stream" field is anything
|
||||
// other than the JSON literal false, JSON null, or an absent key (item
|
||||
// 9/F7 fix, 2026-08-22/23 review). A bare req["stream"].(bool) assertion
|
||||
// silently treats a non-bool value ("stream":"true", "stream":1) as
|
||||
// false and lets it through to an upstream that might itself honor a
|
||||
// loosely-typed truthy value — exactly the "silently answered
|
||||
// non-streamed" outcome this route must reject explicitly instead of
|
||||
// allowing by accident. JSON null is treated the same as an absent key
|
||||
// (F7 fix): an SDK that serializes an unset optional field as explicit
|
||||
// null must not get a hard 400 on an ordinary non-streaming call — null
|
||||
// decodes to a Go nil interface value, which the item-9 fix's "anything
|
||||
// not the literal false" rule would otherwise also reject.
|
||||
func isStreamingRequested(req map[string]any) bool {
|
||||
v, ok := req["stream"]
|
||||
if !ok || v == nil {
|
||||
return false
|
||||
}
|
||||
return v != false
|
||||
}
|
||||
|
||||
// responseTranslationError wraps a failure translating an already-
|
||||
// successful, already-billed upstream response into this route's own
|
||||
// client-facing Anthropic shape (item 1/12 fix, 2026-08-22 review): the
|
||||
// upstream call itself succeeded, so its usage travels back to the
|
||||
// caller as call's own first return value regardless of this error (see
|
||||
// callTranslatedMessages/callAnthropicMessagesPassthrough, below) — a
|
||||
// client who can induce a translation failure (verified repro: a
|
||||
// provider response with usage prompt=5000/completion=900,
|
||||
// finish_reason "length", and a truncated tool-call arguments string)
|
||||
// must not get unlimited free tokens against their budget just because
|
||||
// this gateway's own re-encoding step broke after the provider already
|
||||
// did, and was paid for, the work. Kept distinct from an errUpstream-
|
||||
// wrapped error (a genuine connectivity/protocol failure) so
|
||||
// handleAdapterErrorEnvelope's (routes_unified.go) generic "upstream
|
||||
// connection error" branch never misclassifies a translation bug as a
|
||||
// network problem in the log — see that function's own
|
||||
// *responseTranslationError branch.
|
||||
type responseTranslationError struct {
|
||||
err error
|
||||
}
|
||||
|
||||
// Error implements the error interface.
|
||||
func (e *responseTranslationError) Error() string { return "translate response: " + e.err.Error() }
|
||||
|
||||
// callAnthropicMessagesPassthrough handles a /v1/messages request already
|
||||
// resolved to an anthropic-type provider: forward req to that provider's
|
||||
// own Messages endpoint UNTRANSLATED (brief: "pass through, no
|
||||
// translation of the message body") and copy its response back to w. req's
|
||||
// only mutation, applied by the shared pipeline (runMeteredCall,
|
||||
// routes_unified.go) before this function ever runs, is the "model"
|
||||
// field rewrite to the resolved upstream model id — the same rewrite
|
||||
// every route in this package applies before any adapter call; it is
|
||||
// request ROUTING, not message translation.
|
||||
//
|
||||
// This deliberately bypasses providerAdapter.chatCompletion:
|
||||
// chatCompletion's contract (providers.go) is OpenAI-shape in, OpenAI-
|
||||
// shape out for every provider type — anthropicAdapter.chatCompletion
|
||||
// always calls anthropicRequestFromOpenAI on whatever it is handed, which
|
||||
// would corrupt an already-Anthropic-shaped body instead of passing it
|
||||
// through.
|
||||
//
|
||||
// clientHeaders is the client's own incoming request headers, forwarded
|
||||
// through anthropicPassthroughForwardedHeaders' allowlist only (item 5
|
||||
// fix, above).
|
||||
//
|
||||
// The response body reaches the client BYTE-FOR-BYTE except "model",
|
||||
// rewritten to requestedModel — the client's own requested alias (item 4
|
||||
// fix, 2026-08-22 review: the PREVIOUS version left the upstream's own
|
||||
// real model id in the response, inconsistent with callTranslatedMessages'
|
||||
// own alias echo on the same route, and leaking the operator's real
|
||||
// upstream model id, which gatewayAliasKey exists specifically to
|
||||
// prevent). Byte-for-byte is load-bearing, not a nicety: see
|
||||
// rewriteModelField's own doc comment (item 4/F1 fix, 2026-08-23 review)
|
||||
// for why a decode-then-re-marshal round trip through a Go value
|
||||
// silently corrupts data on this route.
|
||||
func (g *Gateway) callAnthropicMessagesPassthrough(ctx context.Context, w http.ResponseWriter, clientHeaders http.Header, adapter providerAdapter, req map[string]any, requestedModel string) (usage, error) {
|
||||
delete(req, gatewayAliasKey) // this route never sets it via req itself, but a client-sent "__alias" field must never reach the real Anthropic API
|
||||
|
||||
// Plain type assertion, not an added providerAdapter interface method:
|
||||
// this branch only ever runs when the caller already confirmed
|
||||
// adapter.typeName() == providerTypeAnthropic, and buildAdapters
|
||||
// (providers.go) only ever constructs a *anthropicAdapter for that
|
||||
// type, so the assertion is guaranteed to succeed. aa.retry is
|
||||
// nil-safe either way (retryPolicy.do's own contract, retry.go), so a
|
||||
// failed assertion still degrades safely to one-attempt behavior
|
||||
// rather than panicking.
|
||||
aa, _ := adapter.(*anthropicAdapter)
|
||||
var retry *retryPolicy
|
||||
if aa != nil {
|
||||
retry = aa.retry
|
||||
}
|
||||
|
||||
hdr := http.Header{}
|
||||
hdr.Set("Content-Type", "application/json")
|
||||
for _, name := range anthropicPassthroughForwardedHeaders {
|
||||
for _, v := range clientHeaders.Values(name) {
|
||||
hdr.Add(name, v)
|
||||
}
|
||||
}
|
||||
// injectAuth runs AFTER the allowlist copy above, so it always wins
|
||||
// (item 5 ruling) — in practice the two never collide (the allowlist
|
||||
// carries no auth-shaped header), but the ordering is deliberate and
|
||||
// documented, not incidental.
|
||||
adapter.injectAuth(&http.Request{Header: hdr})
|
||||
|
||||
resp, err := upstreamJSON(ctx, adapter.httpClient(), http.MethodPost, adapter.base()+anthropicMessagesPath, hdr, req, retry)
|
||||
if err != nil {
|
||||
return usage{}, err
|
||||
}
|
||||
defer resp.Body.Close() //nolint:errcheck // read-side close; nothing actionable on failure
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return usage{}, newProviderHTTPError(resp)
|
||||
}
|
||||
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
|
||||
if err != nil {
|
||||
// F5 fix, 2026-08-23 review: the upstream already answered 200
|
||||
// (already billed server-side) — a body-read failure past that
|
||||
// point is classified as a translation failure, not a
|
||||
// connectivity problem, consistent with the usage-decode failure
|
||||
// below. Nothing is recoverable here: with no bytes read at all,
|
||||
// there is neither a usage number to extract nor a body to
|
||||
// deliver, so this is still an error return.
|
||||
return usage{}, &responseTranslationError{err: fmt.Errorf("read response body: %w", err)}
|
||||
}
|
||||
|
||||
// F5 fix, 2026-08-23 review: a failure parsing USAGE out of raw must
|
||||
// never block delivering raw itself to the client — the upstream
|
||||
// already fully answered, and this route's whole purpose is passing
|
||||
// that answer through. billed stays usage{} on a parse failure;
|
||||
// runMeteredCall's own zero-usage estimate (from request body size,
|
||||
// routes_unified.go) takes over, exactly like every other
|
||||
// unparseable-usage-shape case already relies on. This is NOT a
|
||||
// repeat of the "silently drop tokens a client can exploit" bug item
|
||||
// 1 fixed elsewhere: there, callErr was returned non-nil and skipped
|
||||
// that estimate fallback entirely; here callErr stays nil, so the
|
||||
// fallback still runs and the request still gets billed something.
|
||||
var parsed anthropicResponseBody
|
||||
var billed usage
|
||||
if unmarshalErr := json.Unmarshal(raw, &parsed); unmarshalErr != nil {
|
||||
g.warnf("messages route: could not decode usage from an anthropic passthrough response (forwarding the response to the client regardless): %v", unmarshalErr)
|
||||
} else {
|
||||
// totalInputTokens folds Anthropic's prompt-cache counters
|
||||
// (cache_creation_input_tokens/cache_read_input_tokens) into the
|
||||
// billed prompt count (item 6 fix, 2026-08-22 review) — see its
|
||||
// own doc comment, translate_anthropic.go, for why dropping them
|
||||
// under-bills a cache-heavy caller (the common case for Claude
|
||||
// Code) by orders of magnitude.
|
||||
billed = usage{prompt: parsed.Usage.totalInputTokens(), completion: parsed.Usage.OutputTokens}
|
||||
}
|
||||
|
||||
out, err := rewriteModelField(raw, requestedModel)
|
||||
if err != nil {
|
||||
return billed, &responseTranslationError{err: err}
|
||||
}
|
||||
|
||||
if ct := resp.Header.Get("Content-Type"); ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
_, _ = w.Write(out)
|
||||
|
||||
return billed, nil
|
||||
}
|
||||
|
||||
// rewriteModelField replaces raw's top-level "model" field with a JSON
|
||||
// string of newModel, preserving every other field's VALUE bytes and
|
||||
// their original relative order exactly (item 4/F1 fix, 2026-08-23
|
||||
// review). A prior version decoded the whole response into
|
||||
// map[string]any and re-marshaled it — not enough: Go's float64 cannot
|
||||
// represent every JSON integer exactly (verified repro:
|
||||
// 12345678901234567890 becomes 12345678901234567000, and
|
||||
// 9007199254740993 becomes 9007199254740992 — both silent corruption of
|
||||
// a value that could be, for instance, a long numeric id inside a
|
||||
// tool_use block's arbitrary "input"), json.Marshal also rewrites 1.0 to
|
||||
// 1 and HTML-escapes < and & by default, and re-marshaling a Go map
|
||||
// always re-sorts its keys regardless of the source's own order — wrong
|
||||
// on a route documented as passthrough.
|
||||
//
|
||||
// This function never turns any value but "model" into a Go value at
|
||||
// all: it walks raw's top-level object with json.Decoder.Token()/Decode()
|
||||
// and captures every other field's value as a json.RawMessage, which
|
||||
// preserves the exact source bytes of that value — including a nested
|
||||
// object or array's own internal structure — without parsing it further.
|
||||
// The rebuilt object is written compactly (no inter-token whitespace):
|
||||
// for the compact JSON every provider in this codebase actually returns
|
||||
// over the wire, that reproduces the input byte-for-byte apart from the
|
||||
// "model" field; a pretty-printed input's insignificant whitespace would
|
||||
// not survive, which no known caller of this function ever produces.
|
||||
func rewriteModelField(raw []byte, newModel string) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(raw))
|
||||
if tok, err := dec.Token(); err != nil || tok != json.Delim('{') {
|
||||
return nil, fmt.Errorf("response is not a JSON object")
|
||||
}
|
||||
|
||||
newModelJSON, err := json.Marshal(newModel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal model field: %w", err)
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
out.WriteByte('{')
|
||||
first := true
|
||||
for dec.More() {
|
||||
keyTok, err := dec.Token()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read field key: %w", err)
|
||||
}
|
||||
key, ok := keyTok.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected non-string object key")
|
||||
}
|
||||
|
||||
var val json.RawMessage
|
||||
if decodeErr := dec.Decode(&val); decodeErr != nil {
|
||||
return nil, fmt.Errorf("read field %q value: %w", key, decodeErr)
|
||||
}
|
||||
|
||||
if !first {
|
||||
out.WriteByte(',')
|
||||
}
|
||||
first = false
|
||||
|
||||
keyJSON, err := json.Marshal(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal field key %q: %w", key, err)
|
||||
}
|
||||
out.Write(keyJSON)
|
||||
out.WriteByte(':')
|
||||
if key == "model" {
|
||||
out.Write(newModelJSON)
|
||||
} else {
|
||||
out.Write(val)
|
||||
}
|
||||
}
|
||||
out.WriteByte('}')
|
||||
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
// callTranslatedMessages handles a /v1/messages request resolved to a
|
||||
// non-anthropic-type provider (openai, or gemini — both implement
|
||||
// providerAdapter.chatCompletion against the SAME OpenAI-shaped
|
||||
// contract, providers.go, regardless of what their own upstream wire
|
||||
// format is): translate req from Anthropic's shape to OpenAI's
|
||||
// (openAIRequestFromAnthropic, translate_anthropic.go), run it through
|
||||
// the resolved adapter's own chatCompletion exactly as routes_unified.go's
|
||||
// handleChat does, capturing what it writes into an in-memory recorder
|
||||
// instead of letting it reach the client directly, then translate that
|
||||
// captured OpenAI-shaped response back to Anthropic's shape
|
||||
// (anthropicResponseFromOpenAI) before writing the result to w.
|
||||
//
|
||||
// USAGE ON A TRANSLATION FAILURE (item 1 fix, 2026-08-22 review):
|
||||
// chatCompletion's own returned usage (result, below) is what the
|
||||
// upstream actually reported and this gateway already billed the
|
||||
// provider for — it is preserved and returned even when translating
|
||||
// THAT successful response into Anthropic's shape fails afterward
|
||||
// (verified repro: a response with usage prompt=5000/completion=900,
|
||||
// finish_reason "length", and a tool-call whose arguments string was
|
||||
// truncated mid-JSON by the provider's own max_tokens cutoff). The
|
||||
// PREVIOUS version of this function returned usage{} on that path,
|
||||
// discarding tokens the provider had already charged for — a client who
|
||||
// could reliably induce a truncated tool-call response got unlimited
|
||||
// free tokens against any budget, and this route was strictly less
|
||||
// robust than /v1/chat/completions, which forwards that same malformed
|
||||
// response to the client successfully instead of erroring on it.
|
||||
func (g *Gateway) callTranslatedMessages(ctx context.Context, w http.ResponseWriter, adapter providerAdapter, req map[string]any, requestedModel string) (usage, error) {
|
||||
openaiReq, dropped, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
return usage{}, err
|
||||
}
|
||||
// thinking (Anthropic's extended-thinking config) and any other
|
||||
// field this translator recognizes but cannot map has no OpenAI
|
||||
// equivalent — logged rather than silently discarded (item 11 fix,
|
||||
// 2026-08-22 review), so an operator can see a client's request was
|
||||
// only partially honored instead of the drop being invisible.
|
||||
for _, field := range dropped {
|
||||
g.warnf("messages route: request field %q has no OpenAI equivalent for provider %q and was dropped in translation", field, adapter.name())
|
||||
}
|
||||
|
||||
rec := newMessagesResponseRecorder()
|
||||
result, err := adapter.chatCompletion(ctx, rec, openaiReq)
|
||||
if err != nil {
|
||||
// result, not usage{} (F5 fix, 2026-08-23 review): harmless today
|
||||
// — streaming is rejected before this route ever reaches here, so
|
||||
// chatCompletion's non-streaming contract already guarantees
|
||||
// usage{} on any error. It matters the day streaming lands: a
|
||||
// streaming chatCompletion call (forwardStream) CAN return
|
||||
// nonzero usage alongside an error — a usage chunk that arrived
|
||||
// just before a dropped connection must still be billed — and
|
||||
// discarding it here would reintroduce item 1's original
|
||||
// CRITICAL (usage silently dropped on an error path) the moment
|
||||
// this route gains streaming support.
|
||||
return result, err
|
||||
}
|
||||
|
||||
out, err := anthropicResponseFromOpenAI(rec.body.Bytes(), requestedModel)
|
||||
if err != nil {
|
||||
return result, &responseTranslationError{err: err}
|
||||
}
|
||||
b, err := json.Marshal(out)
|
||||
if err != nil {
|
||||
return result, &responseTranslationError{err: fmt.Errorf("marshal translated response: %w", err)}
|
||||
}
|
||||
|
||||
status := rec.status
|
||||
if status == 0 {
|
||||
status = http.StatusOK
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write(b)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// messagesResponseRecorder is a minimal in-memory http.ResponseWriter
|
||||
// callTranslatedMessages hands to a resolved adapter's own chatCompletion
|
||||
// in place of the real client connection. chatCompletion's contract
|
||||
// (providerAdapter, providers.go) is OpenAI-shape in, OpenAI-shape out
|
||||
// regardless of provider type, and this route needs to translate that
|
||||
// output to Anthropic's shape before anything reaches the real client —
|
||||
// so the adapter's write is captured here and never forwarded; the real
|
||||
// ResponseWriter only ever sees the translated bytes callTranslatedMessages
|
||||
// writes afterward. Streaming can never reach this recorder: handleMessages
|
||||
// rejects a streaming request before either call path runs, so
|
||||
// chatCompletion's non-streaming forwardJSON is the only path that ever
|
||||
// writes to it.
|
||||
type messagesResponseRecorder struct {
|
||||
header http.Header
|
||||
body bytes.Buffer
|
||||
status int
|
||||
}
|
||||
|
||||
func newMessagesResponseRecorder() *messagesResponseRecorder {
|
||||
return &messagesResponseRecorder{header: http.Header{}}
|
||||
}
|
||||
|
||||
func (r *messagesResponseRecorder) Header() http.Header { return r.header }
|
||||
|
||||
func (r *messagesResponseRecorder) WriteHeader(status int) { r.status = status }
|
||||
|
||||
func (r *messagesResponseRecorder) Write(b []byte) (int, error) {
|
||||
if r.status == 0 {
|
||||
r.status = http.StatusOK // mirrors net/http's implicit-200 default
|
||||
}
|
||||
return r.body.Write(b)
|
||||
}
|
||||
|
||||
// writeAnthropicError writes an Anthropic Messages API-shaped error
|
||||
// envelope: {"type":"error","error":{"type":...,"message":...}} — what an
|
||||
// Anthropic SDK client parses. errType uses the same vocabulary this
|
||||
// gateway's OpenAI-shaped envelope already uses everywhere else
|
||||
// (writeOAIError, errors.go: invalid_request_error, authentication_error,
|
||||
// rate_limit_error, server_error), so an operator reading logs sees one
|
||||
// consistent taxonomy regardless of which route answered; only the JSON
|
||||
// shape around it differs, to match what this route's own clients expect.
|
||||
// Matches envelopeWriter's signature (routes_unified.go), so this route's
|
||||
// error paths thread straight into the shared pipeline there.
|
||||
func writeAnthropicError(w http.ResponseWriter, status int, errType, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
"type": "error",
|
||||
"error": map[string]any{"type": errType, "message": msg},
|
||||
})
|
||||
}
|
||||
|
||||
// writeAnthropicProviderUpstreamError is writeProviderUpstreamError's
|
||||
// (routes_unified.go) Anthropic-shaped counterpart: perr's upstream body
|
||||
// is embedded under error.upstream (decodeUpstreamErrorBody,
|
||||
// routes_unified.go — shared with writeProviderUpstreamError, item 7/8
|
||||
// fix) inside the Anthropic {"type":"error","error":{...}} envelope
|
||||
// instead of the OpenAI one. Matches providerUpstreamErrorWriter's
|
||||
// signature (routes_unified.go).
|
||||
func writeAnthropicProviderUpstreamError(w http.ResponseWriter, providerName string, perr *providerHTTPError) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(perr.status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{ // headers already committed; nothing useful to do on encode failure
|
||||
"type": "error",
|
||||
"error": map[string]any{
|
||||
"type": "upstream_error",
|
||||
"message": providerName + " upstream error",
|
||||
"code": strconv.Itoa(perr.status),
|
||||
"upstream": decodeUpstreamErrorBody(perr),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,681 @@
|
||||
package traefikllmgateway
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newMessagesRequest builds a POST /v1/messages request, JSON-encoding
|
||||
// body. authHeader/authValue let a test choose which credential header to
|
||||
// send; authHeader == "" sends no credential at all.
|
||||
func newMessagesRequest(t *testing.T, body map[string]any, authHeader, authValue string) *http.Request {
|
||||
t.Helper()
|
||||
b, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal request body: %v", err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, messagesPath, bytes.NewReader(b))
|
||||
if authHeader != "" {
|
||||
req.Header.Set(authHeader, authValue)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// newMessagesTestGateway builds a *Gateway wired with an "anthropic"
|
||||
// provider pointed at anthropicURL (skipped when "") and an "openai"
|
||||
// provider pointed at openaiURL (skipped when ""), one permissive
|
||||
// "default" group, and one inline user "alice" — the shared fixture
|
||||
// every handleMessages test below starts from.
|
||||
func newMessagesTestGateway(t *testing.T, anthropicURL, openaiURL string) *Gateway {
|
||||
t.Helper()
|
||||
cfg := CreateConfig()
|
||||
cfg.Providers = map[string]*ProviderConfig{}
|
||||
if anthropicURL != "" {
|
||||
cfg.Providers["anthropic"] = &ProviderConfig{Type: providerTypeAnthropic, BaseURL: anthropicURL, APIKey: "sk-ant-up", Models: []string{"claude-test"}} // #nosec G101 -- test fixture literal, not a real credential
|
||||
}
|
||||
if openaiURL != "" {
|
||||
cfg.Providers["openai"] = &ProviderConfig{Type: providerTypeOpenAI, BaseURL: openaiURL, APIKey: "sk-oai-up", Models: []string{"gpt-test"}}
|
||||
}
|
||||
cfg.Groups = map[string]*GroupConfig{"default": {}}
|
||||
cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{}}}}
|
||||
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
||||
h, err := New(context.Background(), next, cfg, "llmgw")
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
gw, ok := h.(*Gateway)
|
||||
if !ok {
|
||||
t.Fatal("handler is not *Gateway")
|
||||
}
|
||||
return gw
|
||||
}
|
||||
|
||||
// TestHandleMessages_AnthropicProvider_Passthrough proves a /v1/messages
|
||||
// request resolved to an anthropic-type provider is forwarded to that
|
||||
// provider's own Messages endpoint UNTRANSLATED (brief: "pass through, no
|
||||
// translation of the message body") and its response copied back to the
|
||||
// client with every field preserved except "model", which is rewritten to
|
||||
// the client's own requested alias (item 4 fix, 2026-08-22 review: a
|
||||
// routing rewrite, not message-body translation — every OTHER field,
|
||||
// including ones this translator does not itself know about, survives
|
||||
// unchanged), while still authenticating upstream, forwarding the
|
||||
// anthropic-beta header (item 5), billing prompt-cache tokens (item 6),
|
||||
// and accounting usage exactly like every other route.
|
||||
func TestHandleMessages_AnthropicProvider_Passthrough(t *testing.T) {
|
||||
const anthResp = `{"id":"msg_01ABC","type":"message","role":"assistant","content":[{"type":"text","text":"hi there"}],"model":"claude-real","stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":12,"output_tokens":6,"cache_creation_input_tokens":100,"cache_read_input_tokens":50}}`
|
||||
|
||||
var gotBody map[string]any
|
||||
var gotAPIKey, gotVersion, gotPath, gotBeta string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
gotAPIKey = r.Header.Get("x-api-key")
|
||||
gotVersion = r.Header.Get("anthropic-version")
|
||||
gotBeta = r.Header.Get("anthropic-beta")
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotBody)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, srv.URL, "")
|
||||
|
||||
body := map[string]any{
|
||||
"model": "claude-test",
|
||||
"max_tokens": 100,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
}
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
req.Header.Set("anthropic-beta", "token-efficient-tools-2025-02-19")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode client response: %v", err)
|
||||
}
|
||||
if got["model"] != "claude-test" {
|
||||
t.Errorf(`response["model"] = %v, want "claude-test" (the client's requested alias echoed back, item 4 fix — not the upstream's real model id)`, got["model"])
|
||||
}
|
||||
// Every other field must survive the passthrough unchanged, including
|
||||
// ones this gateway does not itself interpret (stop_sequence).
|
||||
if got["id"] != "msg_01ABC" || got["role"] != "assistant" || got["stop_reason"] != "end_turn" {
|
||||
t.Errorf("response id/role/stop_reason = %v/%v/%v, want the upstream's own values preserved", got["id"], got["role"], got["stop_reason"])
|
||||
}
|
||||
if _, hasStopSeq := got["stop_sequence"]; !hasStopSeq {
|
||||
t.Error(`response missing "stop_sequence" — a field this gateway never reads must still survive the passthrough`)
|
||||
}
|
||||
content, _ := got["content"].([]any)
|
||||
if len(content) != 1 {
|
||||
t.Fatalf("response content = %v, want the upstream's one text block preserved", got["content"])
|
||||
}
|
||||
|
||||
if gotPath != anthropicMessagesPath {
|
||||
t.Errorf("upstream path = %q, want %q", gotPath, anthropicMessagesPath)
|
||||
}
|
||||
if gotAPIKey != "sk-ant-up" { // #nosec G101 -- test fixture literal, not a real credential
|
||||
t.Errorf("upstream x-api-key = %q, want sk-ant-up", gotAPIKey)
|
||||
}
|
||||
if gotVersion != anthropicAPIVersion {
|
||||
t.Errorf("upstream anthropic-version = %q, want %q", gotVersion, anthropicAPIVersion)
|
||||
}
|
||||
if gotBeta != "token-efficient-tools-2025-02-19" {
|
||||
t.Errorf("upstream anthropic-beta = %q, want the client's header forwarded (item 5 fix)", gotBeta)
|
||||
}
|
||||
if gotBody["model"] != "claude-test" {
|
||||
t.Errorf("upstream model = %v, want claude-test (resolved upstream id)", gotBody["model"])
|
||||
}
|
||||
if _, hasAlias := gotBody[gatewayAliasKey]; hasAlias {
|
||||
t.Error("upstream request carries gatewayAliasKey; must never reach an upstream provider")
|
||||
}
|
||||
|
||||
// 12 input + 100 cache-creation + 50 cache-read (item 6 fix): all
|
||||
// three fold into the billed prompt count, not just fresh input.
|
||||
tokIn, ok := gw.limiter.getCounter("user", "alice", metricTokIn, windowDay, time.Now())
|
||||
if !ok || tokIn != 162 {
|
||||
t.Errorf("user tokin/day counter = %d (ok=%v), want 162 (12 input + 100 cache-creation + 50 cache-read)", tokIn, ok)
|
||||
}
|
||||
tokOut, ok := gw.limiter.getCounter("user", "alice", metricTokOut, windowDay, time.Now())
|
||||
if !ok || tokOut != 6 {
|
||||
t.Errorf("user tokout/day counter = %d (ok=%v), want 6", tokOut, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_OpenAIProvider_TranslatesBothDirections proves a
|
||||
// /v1/messages request resolved to an openai-type provider is translated
|
||||
// to OpenAI's chat-completion shape going out (Anthropic's top-level
|
||||
// "system" field folded into a role:"system" message) and the upstream's
|
||||
// OpenAI-shaped response translated back to Anthropic's Messages shape
|
||||
// coming back, with usage accounted from the translated numbers.
|
||||
func TestHandleMessages_OpenAIProvider_TranslatesBothDirections(t *testing.T) {
|
||||
const oaiResp = `{"id":"chatcmpl-99","object":"chat.completion","model":"gpt-real","choices":[{"index":0,"message":{"role":"assistant","content":"hi there"},"finish_reason":"stop"}],"usage":{"prompt_tokens":8,"completion_tokens":4}}`
|
||||
|
||||
var gotReq map[string]any
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
_ = json.NewDecoder(r.Body).Decode(&gotReq)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(oaiResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, "", srv.URL)
|
||||
|
||||
body := map[string]any{
|
||||
"model": "gpt-test",
|
||||
"max_tokens": 100,
|
||||
"system": "You are terse.",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
}
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if gotPath != "/v1/chat/completions" {
|
||||
t.Errorf("upstream path = %q, want /v1/chat/completions", gotPath)
|
||||
}
|
||||
|
||||
msgs, _ := gotReq["messages"].([]any)
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("upstream messages = %v, want 2 (system + user)", gotReq["messages"])
|
||||
}
|
||||
first, _ := msgs[0].(map[string]any)
|
||||
if first["role"] != "system" || first["content"] != "You are terse." {
|
||||
t.Errorf("upstream messages[0] = %v, want the system message translated into OpenAI's shape", first)
|
||||
}
|
||||
if _, hasSystem := gotReq["system"]; hasSystem {
|
||||
t.Error("upstream request still carries Anthropic's own \"system\" field; want it folded into messages[]")
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode client response: %v", err)
|
||||
}
|
||||
if got["type"] != "message" || got["role"] != "assistant" {
|
||||
t.Errorf("response type/role = %v/%v, want message/assistant (Anthropic shape)", got["type"], got["role"])
|
||||
}
|
||||
content, _ := got["content"].([]any)
|
||||
if len(content) != 1 {
|
||||
t.Fatalf("response content = %v, want one text block", got["content"])
|
||||
}
|
||||
block, _ := content[0].(map[string]any)
|
||||
if block["type"] != "text" || block["text"] != "hi there" {
|
||||
t.Errorf("response content[0] = %v, want {type: text, text: \"hi there\"}", block)
|
||||
}
|
||||
if got["model"] != "gpt-test" {
|
||||
t.Errorf("response model = %v, want gpt-test (the client's own requested alias echoed back)", got["model"])
|
||||
}
|
||||
if got["stop_reason"] != "end_turn" {
|
||||
t.Errorf("response stop_reason = %v, want end_turn", got["stop_reason"])
|
||||
}
|
||||
|
||||
usageMap, _ := got["usage"].(map[string]any)
|
||||
if usageMap["input_tokens"] != float64(8) || usageMap["output_tokens"] != float64(4) {
|
||||
t.Errorf("response usage = %v, want input_tokens=8 output_tokens=4", usageMap)
|
||||
}
|
||||
|
||||
tokIn, ok := gw.limiter.getCounter("user", "alice", metricTokIn, windowDay, time.Now())
|
||||
if !ok || tokIn != 8 {
|
||||
t.Errorf("user tokin/day counter = %d (ok=%v), want 8", tokIn, ok)
|
||||
}
|
||||
tokOut, ok := gw.limiter.getCounter("user", "alice", metricTokOut, windowDay, time.Now())
|
||||
if !ok || tokOut != 4 {
|
||||
t.Errorf("user tokout/day counter = %d (ok=%v), want 4", tokOut, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_Auth table-drives the three credential cases the
|
||||
// brief calls out explicitly: x-api-key authenticates, Authorization:
|
||||
// Bearer authenticates, and a bad key gets a 401 in ANTHROPIC error
|
||||
// shape ({"type":"error","error":{"type":...}}), not OpenAI's
|
||||
// ({"error":{...}}).
|
||||
func TestHandleMessages_Auth(t *testing.T) {
|
||||
const anthResp = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-real","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
authHeader string
|
||||
authValue string
|
||||
wantStatus int
|
||||
}{
|
||||
{"x-api-key header authenticates", "x-api-key", "sk-alice", http.StatusOK},
|
||||
{"Authorization Bearer header authenticates", "Authorization", "Bearer sk-alice", http.StatusOK},
|
||||
{"bad key is rejected in Anthropic error shape", "x-api-key", "sk-not-a-real-key", http.StatusUnauthorized},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gw := newMessagesTestGateway(t, srv.URL, "")
|
||||
body := map[string]any{"model": "claude-test", "max_tokens": 10, "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
|
||||
req := newMessagesRequest(t, body, tc.authHeader, tc.authValue)
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d, body=%s", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if tc.wantStatus != http.StatusUnauthorized {
|
||||
return
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode error body: %v", err)
|
||||
}
|
||||
if got["type"] != "error" {
|
||||
t.Errorf(`response["type"] = %v, want "error" (Anthropic's envelope shape, not OpenAI's flat one)`, got["type"])
|
||||
}
|
||||
errObj, _ := got["error"].(map[string]any)
|
||||
if errObj["type"] != "authentication_error" {
|
||||
t.Errorf("error.type = %v, want authentication_error", errObj["type"])
|
||||
}
|
||||
if _, hasTopLevelCode := got["code"]; hasTopLevelCode {
|
||||
t.Error(`response carries a top-level "code" field — that is OpenAI's writeOAIError shape, not Anthropic's`)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_StreamingRejected table-drives every "stream" shape
|
||||
// this route must reject cleanly, with a typed, Anthropic-shaped error,
|
||||
// before any upstream call is made or any rate-limit budget is spent —
|
||||
// never silently ignored, never answered non-streamed. Covers a real
|
||||
// boolean true and, per item 9's fix (2026-08-22 review), a non-bool
|
||||
// truthy value ("stream":"true", a string, and "stream":1, a number) that
|
||||
// a bare req["stream"].(bool) assertion would previously have silently
|
||||
// treated as false and forwarded upstream.
|
||||
func TestHandleMessages_StreamingRejected(t *testing.T) {
|
||||
cases := []struct {
|
||||
stream any
|
||||
name string
|
||||
}{
|
||||
{name: "boolean true", stream: true},
|
||||
{name: "string \"true\" (non-bool, item 9)", stream: "true"},
|
||||
{name: "number 1 (non-bool, item 9)", stream: float64(1)},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
called := false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// requestsPerMinute: 1 so a second wrongly-counted request
|
||||
// would 429 — proving the rejection happens before
|
||||
// admitRequest ever runs (item 2 fix): both requests below
|
||||
// must succeed at the "not yet counted" stage.
|
||||
cfg := CreateConfig()
|
||||
cfg.Providers = map[string]*ProviderConfig{"anthropic": {Type: providerTypeAnthropic, BaseURL: srv.URL, APIKey: "sk-ant-up", Models: []string{"claude-test"}}} // #nosec G101 -- test fixture literal, not a real credential
|
||||
cfg.Groups = map[string]*GroupConfig{"default": {}}
|
||||
cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{RequestsPerMinute: 1}}}} // #nosec G101 -- test fixture literal, not a real credential
|
||||
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
||||
h, err := New(context.Background(), next, cfg, "llmgw")
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
gw, ok := h.(*Gateway)
|
||||
if !ok {
|
||||
t.Fatal("handler is not *Gateway")
|
||||
}
|
||||
|
||||
body := map[string]any{"model": "claude-test", "max_tokens": 10, "stream": tc.stream, "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
|
||||
|
||||
for attempt := 1; attempt <= 2; attempt++ {
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
// 400, not 501 (item 2 fix): anthropic-sdk-python's retry
|
||||
// classifies purely on status code and retries anything
|
||||
// >= 500 — a 501 here turns every streaming call into a
|
||||
// three-request SDK retry storm against this route's own
|
||||
// rate limit.
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("attempt %d: status = %d, want 400, body=%s", attempt, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode error body: %v", err)
|
||||
}
|
||||
if got["type"] != "error" {
|
||||
t.Errorf(`response["type"] = %v, want "error"`, got["type"])
|
||||
}
|
||||
errObj, _ := got["error"].(map[string]any)
|
||||
if errObj["type"] != "invalid_request_error" {
|
||||
t.Errorf("error.type = %v, want invalid_request_error", errObj["type"])
|
||||
}
|
||||
msg, _ := errObj["message"].(string)
|
||||
if !strings.Contains(msg, "stream") {
|
||||
t.Errorf("error message = %q, want it to mention streaming is not supported", msg)
|
||||
}
|
||||
}
|
||||
if called {
|
||||
t.Error("upstream was called; streaming must be rejected before any upstream request")
|
||||
}
|
||||
|
||||
// item 2's actual fix: TWO streaming-rejected attempts against
|
||||
// a requestsPerMinute:1 user must BOTH succeed (as 400s, not
|
||||
// as a 429 on the second one) — proving admitRequest's
|
||||
// checkAndCount never ran for either.
|
||||
reqCount, ok := gw.limiter.getCounter("user", "alice", metricReq, windowMin, time.Now())
|
||||
if !ok || reqCount != 0 {
|
||||
t.Errorf("user req:min counter = %d (ok=%v), want 0 — a streaming-rejected request must never consume rate-limit budget", reqCount, ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_NullStream_IsNotStreaming is the regression for F7
|
||||
// (2026-08-23 review): isStreamingRequested's item-9 fix ("anything not
|
||||
// the literal false is streaming") had the side effect of also
|
||||
// rejecting "stream": null, since a bare v != false comparison treats a
|
||||
// JSON null (decoded as a Go nil interface) as "present and not false".
|
||||
// An SDK that serializes an unset optional field as explicit null must
|
||||
// still get an ordinary, successful non-streaming response, not a false
|
||||
// -positive 400.
|
||||
func TestHandleMessages_NullStream_IsNotStreaming(t *testing.T) {
|
||||
const anthResp = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-real","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`
|
||||
called := false
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
called = true
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, srv.URL, "")
|
||||
body := map[string]any{"model": "claude-test", "max_tokens": 10, "stream": nil, "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !called {
|
||||
t.Error("upstream was never called; \"stream\": null must be treated as an ordinary non-streaming request")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_RateLimited_Returns429 proves this route enforces
|
||||
// the caller's group/user requestsPerMinute limit exactly like
|
||||
// routes_unified.go's runUnified: a second request past the limit is
|
||||
// refused before any upstream call, carries a Retry-After header, and
|
||||
// only the first, successful request's usage is accounted.
|
||||
func TestHandleMessages_RateLimited_Returns429(t *testing.T) {
|
||||
upstreamCalls := 0
|
||||
const anthResp = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-real","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
upstreamCalls++
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := CreateConfig()
|
||||
cfg.Providers = map[string]*ProviderConfig{"anthropic": {Type: providerTypeAnthropic, BaseURL: srv.URL, APIKey: "sk-ant-up", Models: []string{"claude-test"}}} // #nosec G101 -- test fixture literal, not a real credential
|
||||
cfg.Groups = map[string]*GroupConfig{"default": {}}
|
||||
cfg.Users = &UsersConfig{Inline: []*UserConfig{{Name: "alice", Group: "default", APIKey: "sk-alice", Limits: &LimitsConfig{RequestsPerMinute: 1}}}}
|
||||
next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})
|
||||
h, err := New(context.Background(), next, cfg, "llmgw")
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
gw, ok := h.(*Gateway)
|
||||
if !ok {
|
||||
t.Fatal("handler is not *Gateway")
|
||||
}
|
||||
|
||||
body := map[string]any{"model": "claude-test", "max_tokens": 10, "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
|
||||
|
||||
req1 := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec1 := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec1, req1)
|
||||
if rec1.Code != http.StatusOK {
|
||||
t.Fatalf("first request status = %d, want 200, body=%s", rec1.Code, rec1.Body.String())
|
||||
}
|
||||
|
||||
req2 := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec2 := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec2, req2)
|
||||
if rec2.Code != http.StatusTooManyRequests {
|
||||
t.Fatalf("second request status = %d, want 429, body=%s", rec2.Code, rec2.Body.String())
|
||||
}
|
||||
if rec2.Header().Get("Retry-After") == "" {
|
||||
t.Error("429 response missing Retry-After header")
|
||||
}
|
||||
if upstreamCalls != 1 {
|
||||
t.Errorf("upstream called %d times, want 1 (second request refused before any upstream call)", upstreamCalls)
|
||||
}
|
||||
|
||||
// Envelope-shape assertion (item 8 fix, 2026-08-22 review): now that
|
||||
// admitRequest threads an envelopeWriter through
|
||||
// (routes_unified.go's writeLimitViolationEnvelope), a 429 on this
|
||||
// route must answer in Anthropic's shape like every other error this
|
||||
// route produces, not OpenAI's flat one.
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec2.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode 429 body: %v", err)
|
||||
}
|
||||
if got["type"] != "error" {
|
||||
t.Errorf(`429 response["type"] = %v, want "error" (Anthropic's envelope shape, not OpenAI's flat one)`, got["type"])
|
||||
}
|
||||
errObj, _ := got["error"].(map[string]any)
|
||||
if errObj["type"] != "rate_limit_error" {
|
||||
t.Errorf("429 error.type = %v, want rate_limit_error", errObj["type"])
|
||||
}
|
||||
if _, hasTopLevelCode := got["code"]; hasTopLevelCode {
|
||||
t.Error(`429 response carries a top-level "code" field — that is OpenAI's writeOAIError shape, not Anthropic's`)
|
||||
}
|
||||
|
||||
tokIn, ok := gw.limiter.getCounter("user", "alice", metricTokIn, windowDay, time.Now())
|
||||
if !ok || tokIn != 1 {
|
||||
t.Errorf("user tokin/day counter = %d (ok=%v), want 1 (only the first, successful request accounted)", tokIn, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_TranslationFailureAfterBilledUsage_PreservesUsage is
|
||||
// the regression for item 1 (CRITICAL, 2026-08-22 review): an
|
||||
// openai-type provider's response can succeed (200, real usage the
|
||||
// provider already charged for) yet still fail THIS route's own
|
||||
// translation into Anthropic's shape — here, a tool-call arguments
|
||||
// string truncated mid-JSON by the provider's own max_tokens cutoff
|
||||
// (finish_reason "length"). The previous version of callTranslatedMessages
|
||||
// returned usage{} on that path, discarding tokens the provider had
|
||||
// already billed — a client who could reliably induce a truncated
|
||||
// tool-call response got unlimited free tokens against any budget. This
|
||||
// proves the client gets a clean 502 (translation genuinely failed, no
|
||||
// valid Anthropic response could be produced) while the usage still
|
||||
// lands in the limiter's counters, and (item 12) that the log line
|
||||
// names the real cause — "response translation failed", never "upstream
|
||||
// connection error", which would misdirect an operator toward a network
|
||||
// problem that never happened.
|
||||
func TestHandleMessages_TranslationFailureAfterBilledUsage_PreservesUsage(t *testing.T) {
|
||||
// Exact verified repro from the review: usage prompt=5000/
|
||||
// completion=900, finish_reason "length", and a tool_calls[].function.
|
||||
// arguments string truncated mid-object.
|
||||
const oaiResp = `{"id":"chatcmpl-trunc","object":"chat.completion","model":"gpt-real","choices":[{"index":0,"message":{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\": \"Lond"}}]},"finish_reason":"length"}],"usage":{"prompt_tokens":5000,"completion_tokens":900}}`
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(oaiResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, "", srv.URL)
|
||||
|
||||
body := map[string]any{
|
||||
"model": "gpt-test",
|
||||
"max_tokens": 100,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
}
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
var rec *httptest.ResponseRecorder
|
||||
logs := captureStderr(t, func() {
|
||||
rec = httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
})
|
||||
|
||||
if rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("status = %d, want 502, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode error body: %v", err)
|
||||
}
|
||||
if got["type"] != "error" {
|
||||
t.Errorf(`response["type"] = %v, want "error" (Anthropic error shape)`, got["type"])
|
||||
}
|
||||
|
||||
// item 1: usage the provider already billed must still land in the
|
||||
// counters, even though the client got an error.
|
||||
tokIn, ok := gw.limiter.getCounter("user", "alice", metricTokIn, windowDay, time.Now())
|
||||
if !ok || tokIn != 5000 {
|
||||
t.Errorf("user tokin/day counter = %d (ok=%v), want 5000 (the upstream's already-billed usage, not discarded on translation failure)", tokIn, ok)
|
||||
}
|
||||
tokOut, ok := gw.limiter.getCounter("user", "alice", metricTokOut, windowDay, time.Now())
|
||||
if !ok || tokOut != 900 {
|
||||
t.Errorf("user tokout/day counter = %d (ok=%v), want 900", tokOut, ok)
|
||||
}
|
||||
|
||||
// item 12: the log must name this a translation failure, never an
|
||||
// upstream connection error — the connection and the upstream call
|
||||
// both succeeded fine.
|
||||
if !strings.Contains(logs, "response translation failed") {
|
||||
t.Errorf("log output = %q, want it to contain \"response translation failed\"", logs)
|
||||
}
|
||||
if strings.Contains(logs, "upstream connection error") {
|
||||
t.Errorf("log output = %q, want it to NOT be classified as an upstream connection error", logs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_AnthropicProvider_Passthrough_PreservesExactBytes is
|
||||
// the regression for F1 (BLOCKING, 2026-08-23 review): the previous
|
||||
// alias-echo fix decoded the upstream's 200 into map[string]any and
|
||||
// re-marshaled it, which silently corrupted data on a route documented
|
||||
// as passthrough — a JSON integer beyond float64's exact-integer range
|
||||
// (verified repro: 12345678901234567890 became 12345678901234567000),
|
||||
// a trailing ".0" on a float (1.0 became 1), and HTML-escaped "<" and
|
||||
// "&". A tool_use block's "input" is arbitrary model-generated JSON, so
|
||||
// a long numeric id inside tool arguments was silently corrupted on
|
||||
// every anthropic-type request through this route. This proves the
|
||||
// fixed version (rewriteModelField, routes_messages.go) reproduces the
|
||||
// upstream body byte-for-byte apart from the "model" field alone: same
|
||||
// field order, same big-integer digits, same "1.0", same literal "<".
|
||||
func TestHandleMessages_AnthropicProvider_Passthrough_PreservesExactBytes(t *testing.T) {
|
||||
const anthResp = `{"id":"msg_01ABC","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"calc","input":{"big_id":12345678901234567890,"score":1.0,"note":"1 < 2 & 3"}}],"model":"claude-real","stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":3}}`
|
||||
const wantBody = `{"id":"msg_01ABC","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"calc","input":{"big_id":12345678901234567890,"score":1.0,"note":"1 < 2 & 3"}}],"model":"claude-test","stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":3}}`
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, srv.URL, "")
|
||||
|
||||
body := map[string]any{
|
||||
"model": "claude-test",
|
||||
"max_tokens": 100,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
|
||||
}
|
||||
req := newMessagesRequest(t, body, "x-api-key", "sk-alice")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Body.String(); got != wantBody {
|
||||
t.Errorf("body = %s\nwant = %s\n(must be byte-for-byte identical apart from the \"model\" field: no float64 rounding, no 1.0->1, no HTML-escaping, no key reordering)", got, wantBody)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleMessages_AnthropicProvider_Passthrough_HeaderAllowlist is the
|
||||
// regression for F3 (BLOCKING, 2026-08-23 review): negative assertions
|
||||
// that a blanket header-forward, or reordering injectAuth ahead of the
|
||||
// allowlist copy while adding x-api-key to it, would both defeat.
|
||||
// Authorization, x-api-key (the CLIENT's own value), and Cookie must
|
||||
// never reach the anthropic-type upstream, and the upstream must see
|
||||
// exactly ONE x-api-key value: the gateway's own configured key
|
||||
// ("sk-ant-up"), never the client's own ("sk-alice"). The exactly-ONE
|
||||
// check matters on its own: injectAuth's Set() call replaces any
|
||||
// existing x-api-key value, so a mutant that adds x-api-key to the
|
||||
// allowlist and reorders injectAuth BEFORE that copy would still leave
|
||||
// Get("x-api-key") returning "sk-ant-up" (Set ran first, Add appended
|
||||
// after) while a second, leaked "sk-alice" value silently rode along —
|
||||
// invisible to a bare .Get() check, which only ever returns the first
|
||||
// value.
|
||||
func TestHandleMessages_AnthropicProvider_Passthrough_HeaderAllowlist(t *testing.T) {
|
||||
const anthResp = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"ok"}],"model":"claude-real","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}`
|
||||
|
||||
var gotHeaders http.Header
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotHeaders = r.Header.Clone()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(anthResp))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
gw := newMessagesTestGateway(t, srv.URL, "")
|
||||
|
||||
body := map[string]any{"model": "claude-test", "max_tokens": 10, "messages": []any{map[string]any{"role": "user", "content": "hi"}}}
|
||||
// Authenticate via Authorization: Bearer (presentedKey, auth.go,
|
||||
// gives it priority over x-api-key when both are present) and
|
||||
// ADDITIONALLY send an x-api-key header of the client's own — unused
|
||||
// for auth here, but still present on the request and must still
|
||||
// never reach the upstream — plus a Cookie header. None of the three
|
||||
// has any business reaching the upstream.
|
||||
req := newMessagesRequest(t, body, "Authorization", "Bearer sk-alice")
|
||||
req.Header.Set("x-api-key", "some-other-client-value") // #nosec G101 -- test fixture literal, not a real credential
|
||||
req.Header.Set("Cookie", "session=client-side-secret")
|
||||
rec := httptest.NewRecorder()
|
||||
gw.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
if got := gotHeaders.Get("Authorization"); got != "" {
|
||||
t.Errorf("upstream Authorization = %q, want empty (never forwarded)", got)
|
||||
}
|
||||
if got := gotHeaders.Get("Cookie"); got != "" {
|
||||
t.Errorf("upstream Cookie = %q, want empty (never forwarded)", got)
|
||||
}
|
||||
apiKeyValues := gotHeaders.Values("x-api-key")
|
||||
if len(apiKeyValues) != 1 {
|
||||
t.Fatalf("upstream x-api-key values = %v, want exactly 1 (the client's own key must never ride along as a second value)", apiKeyValues)
|
||||
}
|
||||
if apiKeyValues[0] != "sk-ant-up" { // #nosec G101 -- test fixture literal, not a real credential
|
||||
t.Errorf("upstream x-api-key = %q, want the gateway's own configured key sk-ant-up, never the client's sk-alice", apiKeyValues[0])
|
||||
}
|
||||
}
|
||||
+12
-1
@@ -426,6 +426,17 @@ type passthroughUsagePayload struct {
|
||||
CompletionTokens int64 `json:"completion_tokens"`
|
||||
InputTokens int64 `json:"input_tokens"`
|
||||
OutputTokens int64 `json:"output_tokens"`
|
||||
// CacheCreationInputTokens/CacheReadInputTokens (item 6 fix,
|
||||
// 2026-08-22 review — applied "for the anthropic provider
|
||||
// generally", not only translate_anthropic.go's own
|
||||
// anthropicUsagePayload): Anthropic's native passthrough
|
||||
// response reports these two prompt-cache counters separately
|
||||
// from InputTokens, and extractPassthroughUsage's own
|
||||
// providerTypeAnthropic case folds them in below so a
|
||||
// cache-heavy caller's usage is not silently under-billed on
|
||||
// this route either.
|
||||
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`
|
||||
CacheReadInputTokens int64 `json:"cache_read_input_tokens"`
|
||||
} `json:"usage"`
|
||||
UsageMetadata struct {
|
||||
PromptTokenCount int64 `json:"promptTokenCount"`
|
||||
@@ -459,7 +470,7 @@ func extractPassthroughUsage(typeName, providerName string, body []byte) (usage,
|
||||
case providerTypeOpenAI:
|
||||
return usage{prompt: payload.Usage.PromptTokens, completion: payload.Usage.CompletionTokens}, model, err
|
||||
case providerTypeAnthropic:
|
||||
return usage{prompt: payload.Usage.InputTokens, completion: payload.Usage.OutputTokens}, model, err
|
||||
return usage{prompt: payload.Usage.InputTokens + payload.Usage.CacheCreationInputTokens + payload.Usage.CacheReadInputTokens, completion: payload.Usage.OutputTokens}, model, err
|
||||
case providerTypeGemini:
|
||||
return usage{prompt: payload.UsageMetadata.PromptTokenCount, completion: payload.UsageMetadata.CandidatesTokenCount}, model, err
|
||||
default:
|
||||
|
||||
@@ -69,6 +69,22 @@ func TestExtractPassthroughUsage(t *testing.T) {
|
||||
wantUsage: usage{prompt: 11, completion: 4},
|
||||
wantModel: "claude-native",
|
||||
},
|
||||
{
|
||||
// F2 fix (BLOCKING, 2026-08-23 review): reverting
|
||||
// extractPassthroughUsage's providerTypeAnthropic case back
|
||||
// to input_tokens alone (dropping the cache_creation_input_
|
||||
// tokens/cache_read_input_tokens terms) must fail this case —
|
||||
// this is the production /{provider}/... native passthrough
|
||||
// route, and the previous version silently under-billed a
|
||||
// cache-heavy tenant by orders of magnitude with zero test
|
||||
// coverage catching it.
|
||||
name: "anthropic shape with prompt-cache tokens billed (F2)",
|
||||
typeName: providerTypeAnthropic,
|
||||
providerName: "anthropic",
|
||||
body: `{"model":"claude-native","usage":{"input_tokens":4,"cache_creation_input_tokens":180000,"cache_read_input_tokens":20000,"output_tokens":9}}`,
|
||||
wantUsage: usage{prompt: 200004, completion: 9},
|
||||
wantModel: "claude-native",
|
||||
},
|
||||
{
|
||||
name: "gemini shape, no model field",
|
||||
typeName: providerTypeGemini,
|
||||
|
||||
+219
-93
@@ -116,8 +116,29 @@ func (g *Gateway) handleModels(w http.ResponseWriter, r *http.Request) {
|
||||
// handleChat and handleEmbeddings share every other step of the pipeline
|
||||
// below (decode, resolve, enforce limits, account usage, translate
|
||||
// errors) and differ only in which of these two methods they invoke.
|
||||
// routes_messages.go's handleMessages reuses the identical contract for
|
||||
// its own anthropic-passthrough/translate branching (see its own doc
|
||||
// comment) — the shape is exactly what runMeteredCall, below, needs
|
||||
// regardless of which wire shape a route answers in.
|
||||
type adapterCall func(a providerAdapter, ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error)
|
||||
|
||||
// envelopeWriter abstracts the client-facing error envelope shape so the
|
||||
// shared metered-request pipeline (runMeteredCall, below) can serve both
|
||||
// the OpenAI-shaped unified/media routes and the Anthropic-shaped
|
||||
// /v1/messages route through one implementation (item 7, 2026-08-22
|
||||
// review — fixing 58 of handleMessages' 73 lines being verbatim
|
||||
// duplicated from runUnified) rather than duplicating it. writeOAIError
|
||||
// (errors.go) and writeAnthropicError (routes_messages.go) both already
|
||||
// match this exact signature.
|
||||
type envelopeWriter func(w http.ResponseWriter, status int, errType, msg string)
|
||||
|
||||
// providerUpstreamErrorWriter abstracts which envelope shape a
|
||||
// *providerHTTPError gets embedded into (writeProviderUpstreamError,
|
||||
// below, or routes_messages.go's writeAnthropicProviderUpstreamError) —
|
||||
// threaded through runMeteredCall/handleAdapterErrorEnvelope alongside
|
||||
// envelopeWriter for the same reason.
|
||||
type providerUpstreamErrorWriter func(w http.ResponseWriter, providerName string, perr *providerHTTPError)
|
||||
|
||||
// handleChat implements POST /v1/chat/completions.
|
||||
func (g *Gateway) handleChat(w http.ResponseWriter, r *http.Request, u *user, grp *group) {
|
||||
g.runUnified(w, r, u, grp, cacheEndpointChat, func(a providerAdapter, ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error) {
|
||||
@@ -134,15 +155,14 @@ func (g *Gateway) handleEmbeddings(w http.ResponseWriter, r *http.Request, u *us
|
||||
|
||||
// runUnified is the shared chat/embeddings pipeline: enforce per-user/
|
||||
// per-group/total REQUEST-RATE limits, claim a body-admission slot to
|
||||
// decode the request, resolve its model, invoke the adapter via call,
|
||||
// then account the resulting usage — even when call itself returned an
|
||||
// error, so usage captured before a mid-stream failure still gets billed
|
||||
// — before translating that error into a response. w is wrapped in its
|
||||
// own statusTrackingWriter so a mid-stream adapter error (headers
|
||||
// already sent) can be told apart from one that failed before any
|
||||
// write. endpoint is cacheEndpointChat or cacheEndpointEmbeddings — one
|
||||
// of cacheKey's key-material components (cache.go), so the two routes
|
||||
// never collide into one cache entry.
|
||||
// decode the request, then hand off to runMeteredCall (below) for
|
||||
// everything from model resolution through response caching, usage
|
||||
// accounting, and adapter-error handling. w is wrapped in its own
|
||||
// statusTrackingWriter so a mid-stream adapter error (headers already
|
||||
// sent) can be told apart from one that failed before any write.
|
||||
// endpoint is cacheEndpointChat or cacheEndpointEmbeddings — one of
|
||||
// cacheKey's key-material components (cache.go), so the two routes never
|
||||
// collide into one cache entry.
|
||||
//
|
||||
// ORDERING (security review finding 1a, 2026-08-22): the rate-limit
|
||||
// check runs BEFORE the body is ever read, not after — the previous
|
||||
@@ -182,15 +202,26 @@ func (g *Gateway) handleEmbeddings(w http.ResponseWriter, r *http.Request, u *us
|
||||
// cannot even form a valid request is now itself rate-limited rather
|
||||
// than getting unlimited free retries — but are called out explicitly
|
||||
// here since neither is visible from the diff alone.
|
||||
//
|
||||
// SHARED CORE (item 7 fix, 2026-08-22 review): routes_messages.go's
|
||||
// handleMessages needs admitRequest and the body decode in a DIFFERENT
|
||||
// relative order than this route does (its own doc comment explains
|
||||
// why — a streaming request must never touch the rate-limit counters at
|
||||
// all, which means it has to know "is this streaming" from the decoded
|
||||
// body before admitRequest ever runs). That is the one genuine ordering
|
||||
// difference between the two routes, and it is why THIS function still
|
||||
// owns admitRequest/readAndDecodeUnifiedBody directly rather than folding
|
||||
// them into runMeteredCall too — everything after both are done is
|
||||
// identical regardless of wire shape, and lives in runMeteredCall alone.
|
||||
func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, grp *group, endpoint string, call adapterCall) {
|
||||
sw := &statusTrackingWriter{ResponseWriter: w}
|
||||
|
||||
scopes, ok := g.admitRequest(sw, u, grp)
|
||||
scopes, ok := g.admitRequest(sw, u, grp, writeOAIError)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
body, req, ok := g.readAndDecodeUnifiedBody(sw, r)
|
||||
body, req, ok := g.readAndDecodeUnifiedBody(sw, r, writeOAIError)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
@@ -200,11 +231,35 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
writeOAIError(sw, http.StatusBadRequest, "invalid_request_error", "model is required")
|
||||
return
|
||||
}
|
||||
streaming, _ := req["stream"].(bool)
|
||||
|
||||
g.runMeteredCall(sw, r, scopes, body, req, requestedModel, grp, endpoint, "unified route", writeOAIError, writeProviderUpstreamError, call)
|
||||
}
|
||||
|
||||
// runMeteredCall is the shared implementation from model resolution
|
||||
// through response caching, usage accounting, and adapter-error handling
|
||||
// — the part of every metered LLM route (this file's runUnified, above;
|
||||
// routes_messages.go's handleMessages) that is genuinely identical
|
||||
// regardless of wire shape (item 7 fix, 2026-08-22 review: this body used
|
||||
// to be duplicated almost verbatim in routes_messages.go). Callers have
|
||||
// already run admitRequest (rate limiting) and decoded the request body
|
||||
// into req, in whichever relative order their own route needs (see
|
||||
// runUnified's own doc comment for why that order differs for
|
||||
// /v1/messages) — this function starts only once both are done.
|
||||
//
|
||||
// endpoint is cacheKey's endpoint argument (cacheEndpointChat/
|
||||
// cacheEndpointEmbeddings/routes_messages.go's cacheEndpointMessages).
|
||||
// envelope and writeUpstream pick the client-facing error shape. call is
|
||||
// adapterCall — how the wire-shape-specific request/response translation,
|
||||
// if any, actually happens; the routes differ ONLY here (call's own
|
||||
// closure already has whatever it needs — routes_messages.go's own
|
||||
// requestedModel for its alias echo, in particular) and in which error
|
||||
// envelope shape they answer in. logPrefix names the route in every log
|
||||
// line this function writes, so a log reader can always tell which route
|
||||
// produced a given line without the classification logic existing twice.
|
||||
func (g *Gateway) runMeteredCall(sw *statusTrackingWriter, r *http.Request, scopes []limitScope, body []byte, req map[string]any, requestedModel string, grp *group, endpoint, logPrefix string, envelope envelopeWriter, writeUpstream providerUpstreamErrorWriter, call adapterCall) {
|
||||
adapter, upstreamModel, canonical, err := g.registry.resolve(requestedModel, grp)
|
||||
if err != nil {
|
||||
writeModelResolveError(sw, err)
|
||||
writeModelResolveErrorEnvelope(sw, err, envelope)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -224,6 +279,7 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
// clients requesting the same upstream model under different alias
|
||||
// forms (e.g. "claude-x" vs "anthropic/claude-x") must never collide
|
||||
// into one cache entry (cacheKey's own doc comment, cache.go).
|
||||
streaming, _ := req["stream"].(bool)
|
||||
cacheable := !streaming && g.cache != nil && groupCacheEnabled(grp)
|
||||
var cacheKeyStr string
|
||||
if cacheable {
|
||||
@@ -252,8 +308,8 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
// respWriter is sw, wrapped in a capture tee only when this request
|
||||
// is cacheable — cacheCaptureWriter buffers everything written so a
|
||||
// 200 non-stream response can be stored after call() returns, without
|
||||
// any adapter knowing caching exists (routes_unified.go's adapterCall
|
||||
// contract is unchanged either way).
|
||||
// any adapter knowing caching exists (adapterCall's contract is
|
||||
// unchanged either way).
|
||||
var respWriter http.ResponseWriter = sw
|
||||
var capture *cacheCaptureWriter
|
||||
if cacheable {
|
||||
@@ -280,9 +336,14 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
// translateError, or a connection failure before any write) always
|
||||
// carries zero usage by contract, so accounting it here is a no-op —
|
||||
// account skips every write once both total tokens and cost are zero.
|
||||
// routes_messages.go's callTranslatedMessages/
|
||||
// callAnthropicMessagesPassthrough follow the identical contract for
|
||||
// their own error paths (item 1 fix, 2026-08-22 review: a translation
|
||||
// failure AFTER a successful, billed upstream call must still return
|
||||
// that call's real usage, not usage{}).
|
||||
if callErr == nil && result.total() == 0 {
|
||||
if streaming {
|
||||
g.logf("unified route: zero usage reported for a streaming response from model %q; accounting the request only", canonical)
|
||||
g.logf("%s: zero usage reported for a streaming response from model %q; accounting the request only", logPrefix, canonical)
|
||||
} else {
|
||||
result.prompt = int64(math.Ceil(float64(len(body)) / 4))
|
||||
result.estimated = true
|
||||
@@ -290,7 +351,7 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
}
|
||||
g.limiter.account(scopes, result, unifiedCostMicros(canonical, upstreamModel, result, g.cfg.Pricing))
|
||||
if result.estimated {
|
||||
g.logf("unified route: usage for model %q logged as estimated (%d prompt tokens derived from request body size, not the provider's reported usage)", canonical, result.prompt)
|
||||
g.logf("%s: usage for model %q logged as estimated (%d prompt tokens derived from request body size, not the provider's reported usage)", logPrefix, canonical, result.prompt)
|
||||
}
|
||||
|
||||
// A miss stores the response after everything above has already run —
|
||||
@@ -310,7 +371,7 @@ func (g *Gateway) runUnified(w http.ResponseWriter, r *http.Request, u *user, gr
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
g.handleAdapterError(sw, callErr, adapter.name())
|
||||
g.handleAdapterErrorEnvelope(sw, callErr, adapter.name(), logPrefix, envelope, writeUpstream)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -385,19 +446,23 @@ func withTotalScope(scopes []limitScope) []limitScope {
|
||||
// caller can reuse the identical scope slice for its own later account
|
||||
// call, rather than rebuilding (and risking scope-list drift from) a
|
||||
// second buildLimitScopes/withTotalScope pair. A violation writes its
|
||||
// response to sw itself (writeLimitViolation, unchanged body/headers from
|
||||
// before this fix) and returns ok=false.
|
||||
// response to sw itself, via envelope (writeLimitViolationEnvelope,
|
||||
// below), and returns ok=false.
|
||||
//
|
||||
// Shared verbatim by every unified/media route (runUnified above;
|
||||
// handleImagesGenerations/handleAudioSpeech/handleAudioTranscriptions,
|
||||
// routes_media.go) — see runUnified's own "BEHAVIOR CHANGE" doc-comment
|
||||
// Shared verbatim by every unified/media/messages route (runUnified
|
||||
// above; handleImagesGenerations/handleAudioSpeech/
|
||||
// handleAudioTranscriptions, routes_media.go; handleMessages,
|
||||
// routes_messages.go) — see runUnified's own "BEHAVIOR CHANGE" doc-comment
|
||||
// section for the two previously-silent side effects this reordering has
|
||||
// for a caller who is both over budget and sending a malformed/
|
||||
// unroutable request; they apply identically here.
|
||||
func (g *Gateway) admitRequest(sw *statusTrackingWriter, u *user, grp *group) (scopes []limitScope, ok bool) {
|
||||
// unroutable request; they apply identically here. envelope (item 7 fix,
|
||||
// 2026-08-22 review) lets routes_messages.go answer a violation in
|
||||
// Anthropic's own error shape instead of OpenAI's, without a second copy
|
||||
// of this function's own logic.
|
||||
func (g *Gateway) admitRequest(sw *statusTrackingWriter, u *user, grp *group, envelope envelopeWriter) (scopes []limitScope, ok bool) {
|
||||
scopes = withTotalScope(buildLimitScopes(u, grp))
|
||||
if violation := g.limiter.checkAndCount(scopes); violation != nil {
|
||||
writeLimitViolation(sw, violation)
|
||||
writeLimitViolationEnvelope(sw, violation, envelope)
|
||||
return scopes, false
|
||||
}
|
||||
return scopes, true
|
||||
@@ -457,7 +522,8 @@ const bodyAdmissionRetryAfterSeconds = 2
|
||||
// of goroutines blocked on a channel receive) — it fails immediately
|
||||
// instead, writing a 503 with a Retry-After header to sw and returning a
|
||||
// no-op release so the caller's own `defer release()` stays valid
|
||||
// either way.
|
||||
// either way. envelope (item 7 fix, 2026-08-22 review) picks the JSON
|
||||
// shape that 503 gets written in.
|
||||
//
|
||||
// release must be deferred by the caller IMMEDIATELY after this
|
||||
// returns, BEFORE checking ok, and that defer must be scoped narrowly
|
||||
@@ -477,13 +543,13 @@ const bodyAdmissionRetryAfterSeconds = 2
|
||||
// this semaphore to bound — the only body-inspection it ever does is
|
||||
// peekPassthroughModel's own bounded 64KiB read, an intentional
|
||||
// omission, not an oversight.
|
||||
func (g *Gateway) acquireBodyAdmission(sw *statusTrackingWriter) (release func(), ok bool) {
|
||||
func (g *Gateway) acquireBodyAdmission(sw *statusTrackingWriter, envelope envelopeWriter) (release func(), ok bool) {
|
||||
select {
|
||||
case g.bodyAdmission <- struct{}{}:
|
||||
return func() { <-g.bodyAdmission }, true
|
||||
default:
|
||||
sw.Header().Set("Retry-After", strconv.Itoa(bodyAdmissionRetryAfterSeconds))
|
||||
writeOAIError(sw, http.StatusServiceUnavailable, "server_error", "server is at capacity; try again shortly")
|
||||
envelope(sw, http.StatusServiceUnavailable, "server_error", "server is at capacity; try again shortly")
|
||||
return func() {}, false
|
||||
}
|
||||
}
|
||||
@@ -494,14 +560,16 @@ func (g *Gateway) acquireBodyAdmission(sw *statusTrackingWriter) (release func()
|
||||
// successful decode, a read failure, a decode failure, or admission
|
||||
// exhaustion itself (security review finding 1b, round 3, 2026-08-22).
|
||||
// This is deliberately the ENTIRE scope of what the semaphore guards for
|
||||
// the unified routes: by the time this returns, req is fully decoded and
|
||||
// nothing further in runUnified re-amplifies memory the way the decode
|
||||
// the unified/messages routes: by the time this returns, req is fully
|
||||
// decoded and nothing further re-amplifies memory the way the decode
|
||||
// itself does, so model resolution, the cache lookup, and the upstream
|
||||
// round trip all run after the slot is already released. A read or
|
||||
// decode failure has already written its 400 to sw; ok reports whether
|
||||
// the caller may proceed.
|
||||
func (g *Gateway) readAndDecodeUnifiedBody(sw *statusTrackingWriter, r *http.Request) (body []byte, req map[string]any, ok bool) {
|
||||
release, admitted := g.acquireBodyAdmission(sw)
|
||||
// decode failure has already written its 400 to sw, through envelope
|
||||
// (item 7 fix, 2026-08-22 review — routes_messages.go passes
|
||||
// writeAnthropicError here so this shared step answers in this route's
|
||||
// own error shape); ok reports whether the caller may proceed.
|
||||
func (g *Gateway) readAndDecodeUnifiedBody(sw *statusTrackingWriter, r *http.Request, envelope envelopeWriter) (body []byte, req map[string]any, ok bool) {
|
||||
release, admitted := g.acquireBodyAdmission(sw, envelope)
|
||||
defer release()
|
||||
if !admitted {
|
||||
return nil, nil, false
|
||||
@@ -509,12 +577,12 @@ func (g *Gateway) readAndDecodeUnifiedBody(sw *statusTrackingWriter, r *http.Req
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, maxUnifiedRequestBytes))
|
||||
if err != nil {
|
||||
writeOAIError(sw, http.StatusBadRequest, "invalid_request_error", "cannot read request body")
|
||||
envelope(sw, http.StatusBadRequest, "invalid_request_error", "cannot read request body")
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(body, &req); err != nil {
|
||||
writeOAIError(sw, http.StatusBadRequest, "invalid_request_error", "invalid JSON body")
|
||||
envelope(sw, http.StatusBadRequest, "invalid_request_error", "invalid JSON body")
|
||||
return nil, nil, false
|
||||
}
|
||||
return body, req, true
|
||||
@@ -536,58 +604,79 @@ func unifiedCostMicros(canonical, bare string, u usage, overrides map[string]*Mo
|
||||
return costMicros(bare, u, overrides)
|
||||
}
|
||||
|
||||
// writeModelResolveError maps a modelRegistry.resolve error to its HTTP
|
||||
// envelope: errModelUnknown is a 404 (no configured provider knows this
|
||||
// model), errModelDenied is a 403 (a real model the caller's group cannot
|
||||
// use). A *aliasTargetError (spec §5, v0.2) is also a 404, but with its
|
||||
// own message naming both the alias and its unresolved target, checked
|
||||
// first via a plain type assertion — not errors.As, matching this
|
||||
// package's established yaegi-safe convention for a pointer error type
|
||||
// (registry.go's aliasTargetError doc comment). Any other error is a
|
||||
// defensive 500 — resolve's own contract promises only these sentinels/
|
||||
// types, so reaching this branch would be a programming error, not a
|
||||
// client mistake.
|
||||
func writeModelResolveError(w http.ResponseWriter, err error) {
|
||||
// writeModelResolveErrorEnvelope maps a modelRegistry.resolve error to
|
||||
// its HTTP status/message, written through whichever envelope the caller
|
||||
// supplies (item 7 fix, 2026-08-22 review): errModelUnknown is a 404 (no
|
||||
// configured provider knows this model), errModelDenied is a 403 (a real
|
||||
// model the caller's group cannot use). A *aliasTargetError (spec §5,
|
||||
// v0.2) is also a 404, but with its own message naming both the alias and
|
||||
// its unresolved target, checked first via a plain type assertion — not
|
||||
// errors.As, matching this package's established yaegi-safe convention
|
||||
// for a pointer error type (registry.go's aliasTargetError doc comment).
|
||||
// Any other error is a defensive 500 — resolve's own contract promises
|
||||
// only these sentinels/types, so reaching this branch would be a
|
||||
// programming error, not a client mistake.
|
||||
func writeModelResolveErrorEnvelope(w http.ResponseWriter, err error, envelope envelopeWriter) {
|
||||
if aerr, ok := err.(*aliasTargetError); ok {
|
||||
writeOAIError(w, http.StatusNotFound, "invalid_request_error", aerr.Error())
|
||||
envelope(w, http.StatusNotFound, "invalid_request_error", aerr.Error())
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, errModelUnknown):
|
||||
writeOAIError(w, http.StatusNotFound, "invalid_request_error", "unknown model")
|
||||
envelope(w, http.StatusNotFound, "invalid_request_error", "unknown model")
|
||||
case errors.Is(err, errModelDenied):
|
||||
writeOAIError(w, http.StatusForbidden, "invalid_request_error", "model access denied")
|
||||
envelope(w, http.StatusForbidden, "invalid_request_error", "model access denied")
|
||||
default:
|
||||
writeOAIError(w, http.StatusInternalServerError, "server_error", "internal error")
|
||||
envelope(w, http.StatusInternalServerError, "server_error", "internal error")
|
||||
}
|
||||
}
|
||||
|
||||
// writeLimitViolation maps a limiter violation to its HTTP envelope
|
||||
// (ruling b): a storeDown violation — the configured limit store was
|
||||
// unreachable and failOpen is false — is a 503 server_error, since it is
|
||||
// not an actual limit breach; any other violation is a 429 rate_limit_error
|
||||
// carrying v's own message, plus a Retry-After header when the violation
|
||||
// names a meaningful retry window.
|
||||
func writeLimitViolation(w http.ResponseWriter, v *limitViolation) {
|
||||
// writeModelResolveError is writeModelResolveErrorEnvelope pinned to the
|
||||
// OpenAI envelope shape — the thin wrapper every existing OpenAI-shaped
|
||||
// call site (this file's runMeteredCall, via writeOAIError;
|
||||
// routes_media.go's resolveMediaModel) keeps calling unchanged.
|
||||
func writeModelResolveError(w http.ResponseWriter, err error) {
|
||||
writeModelResolveErrorEnvelope(w, err, writeOAIError)
|
||||
}
|
||||
|
||||
// writeLimitViolationEnvelope maps a limiter violation to its HTTP
|
||||
// status/message, written through whichever envelope the caller supplies
|
||||
// (item 7 fix, 2026-08-22 review): a storeDown violation — the configured
|
||||
// limit store was unreachable and failOpen is false — is a 503
|
||||
// server_error, since it is not an actual limit breach; any other
|
||||
// violation is a 429 rate_limit_error carrying v's own message, plus a
|
||||
// Retry-After header when the violation names a meaningful retry window.
|
||||
func writeLimitViolationEnvelope(w http.ResponseWriter, v *limitViolation, envelope envelopeWriter) {
|
||||
if v.storeDown {
|
||||
writeOAIError(w, http.StatusServiceUnavailable, "server_error", v.message)
|
||||
envelope(w, http.StatusServiceUnavailable, "server_error", v.message)
|
||||
return
|
||||
}
|
||||
if v.retryAfter > 0 {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(v.retryAfter))
|
||||
}
|
||||
writeOAIError(w, http.StatusTooManyRequests, "rate_limit_error", v.message)
|
||||
envelope(w, http.StatusTooManyRequests, "rate_limit_error", v.message)
|
||||
}
|
||||
|
||||
// handleAdapterError translates an adapter's returned error into a
|
||||
// response, or into nothing at all, per ruling c. sw.wroteHeader is
|
||||
// checked first, ahead of every other case: an error surfacing after the
|
||||
// adapter already started writing a response (a mid-stream connection
|
||||
// drop) must never get a second, conflicting envelope appended, whatever
|
||||
// kind of error it is.
|
||||
func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, providerName string) {
|
||||
// writeLimitViolation is writeLimitViolationEnvelope pinned to the
|
||||
// OpenAI envelope shape — the thin wrapper every existing direct call
|
||||
// site (mcp_a2a.go, mcp_federation.go, routes_passthrough.go, all of
|
||||
// which call checkAndCount themselves rather than through admitRequest)
|
||||
// keeps calling unchanged.
|
||||
func writeLimitViolation(w http.ResponseWriter, v *limitViolation) {
|
||||
writeLimitViolationEnvelope(w, v, writeOAIError)
|
||||
}
|
||||
|
||||
// handleAdapterErrorEnvelope is the shared adapter-error classification
|
||||
// every metered route goes through (item 7 fix, 2026-08-22 review):
|
||||
// sw.wroteHeader is checked first, ahead of every other case, since an
|
||||
// error surfacing after the adapter already started writing a response
|
||||
// (a mid-stream connection drop) must never get a second, conflicting
|
||||
// envelope appended, whatever kind of error it is. envelope/writeUpstream
|
||||
// pick the client-facing shape; logPrefix names the route in every log
|
||||
// line.
|
||||
func (g *Gateway) handleAdapterErrorEnvelope(sw *statusTrackingWriter, err error, providerName, logPrefix string, envelope envelopeWriter, writeUpstream providerUpstreamErrorWriter) {
|
||||
// A cacheable request's miss path pre-sets X-Llmgw-Cache: miss before
|
||||
// call() runs (runUnified), so headers precede a successful body —
|
||||
// call() runs (runMeteredCall), so headers precede a successful body —
|
||||
// but an adapter error means nothing was actually served from, or
|
||||
// stored to, the cache. Del is unconditional and harmless when the
|
||||
// header was never set (a no-op on an absent key), so this needs no
|
||||
@@ -597,7 +686,7 @@ func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, provid
|
||||
sw.Header().Del("X-Llmgw-Cache")
|
||||
|
||||
if sw.wroteHeader {
|
||||
g.errorf("unified route: adapter error after response started (provider %q): %v", providerName, err)
|
||||
g.errorf("%s: adapter error after response started (provider %q): %v", logPrefix, providerName, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -611,7 +700,7 @@ func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, provid
|
||||
// Verified empirically under real Traefik (Task 15's integration
|
||||
// suite); tools/yaegi-check never exercises this call path.
|
||||
if perr, ok := err.(*providerHTTPError); ok {
|
||||
writeProviderUpstreamError(sw, providerName, perr)
|
||||
writeUpstream(sw, providerName, perr)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -620,17 +709,58 @@ func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, provid
|
||||
if terr.notSupported {
|
||||
status = http.StatusNotImplemented
|
||||
}
|
||||
writeOAIError(sw, status, "invalid_request_error", terr.msg)
|
||||
envelope(sw, status, "invalid_request_error", terr.msg)
|
||||
return
|
||||
}
|
||||
|
||||
// *responseTranslationError (routes_messages.go, item 1/12 fix,
|
||||
// 2026-08-22 review) is the /v1/messages route's own: a translation
|
||||
// failure AFTER a successful, already-billed upstream call (its usage
|
||||
// already travels back through call's own return value regardless of
|
||||
// this branch — see runMeteredCall's own comment on that). This
|
||||
// branch only ever fires for routes_messages.go's own call closures;
|
||||
// runUnified's adapterCall implementations never produce this type,
|
||||
// so it is always a no-op miss for that route. Kept here, in the one
|
||||
// shared classification, rather than a second copy of this whole
|
||||
// cascade in routes_messages.go, so the classification logic itself
|
||||
// never exists twice — see handleAdapterErrorEnvelope's own doc
|
||||
// comment (item 7).
|
||||
if terr, ok := err.(*responseTranslationError); ok {
|
||||
g.errorf("%s: response translation failed after a billed upstream call (provider %q): %v", logPrefix, providerName, terr.err)
|
||||
envelope(sw, http.StatusBadGateway, "server_error", "failed to translate provider response")
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, context.Canceled) {
|
||||
g.logf("unified route: client canceled request to provider %q: %v", providerName, err)
|
||||
g.logf("%s: client canceled request to provider %q: %v", logPrefix, providerName, err)
|
||||
return
|
||||
}
|
||||
|
||||
g.errorf("unified route: upstream connection error (provider %q): %v", providerName, err)
|
||||
writeOAIError(sw, http.StatusBadGateway, "server_error", "upstream connection error")
|
||||
g.errorf("%s: upstream connection error (provider %q): %v", logPrefix, providerName, err)
|
||||
envelope(sw, http.StatusBadGateway, "server_error", "upstream connection error")
|
||||
}
|
||||
|
||||
// handleAdapterError is handleAdapterErrorEnvelope pinned to the OpenAI
|
||||
// envelope shape — the thin wrapper every existing OpenAI-shaped call
|
||||
// site (routes_media.go's three media handlers) keeps calling unchanged.
|
||||
// runMeteredCall (this file) calls handleAdapterErrorEnvelope directly,
|
||||
// with whichever envelope its own caller supplied.
|
||||
func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, providerName string) {
|
||||
g.handleAdapterErrorEnvelope(sw, err, providerName, "unified route", writeOAIError, writeProviderUpstreamError)
|
||||
}
|
||||
|
||||
// decodeUpstreamErrorBody decodes perr.body as JSON when it parses as
|
||||
// one, or returns it as a raw string otherwise — the shared read
|
||||
// writeProviderUpstreamError and routes_messages.go's
|
||||
// writeAnthropicProviderUpstreamError both embed under "upstream" in
|
||||
// their own envelope shape (item 7/8 fix, 2026-08-22 review).
|
||||
func decodeUpstreamErrorBody(perr *providerHTTPError) any {
|
||||
var upstream any = string(perr.body)
|
||||
var parsed any
|
||||
if json.Unmarshal(perr.body, &parsed) == nil {
|
||||
upstream = parsed
|
||||
}
|
||||
return upstream
|
||||
}
|
||||
|
||||
// writeProviderUpstreamError passes perr's status through to the client,
|
||||
@@ -638,16 +768,10 @@ func (g *Gateway) handleAdapterError(sw *statusTrackingWriter, err error, provid
|
||||
// verbatim (unified-route callers get this wrapped form; the openai-type
|
||||
// adapter's own passthrough forwards a non-2xx body unwrapped, since that
|
||||
// path never reaches this function). perr.body is embedded under
|
||||
// error.upstream, decoded to a JSON value when it parses as one and left
|
||||
// as a raw string otherwise, so a caller sees exactly what the provider
|
||||
// said either way.
|
||||
// error.upstream (decodeUpstreamErrorBody, above), decoded to a JSON
|
||||
// value when it parses as one and left as a raw string otherwise, so a
|
||||
// caller sees exactly what the provider said either way.
|
||||
func writeProviderUpstreamError(w http.ResponseWriter, providerName string, perr *providerHTTPError) {
|
||||
var upstream any = string(perr.body)
|
||||
var parsed any
|
||||
if json.Unmarshal(perr.body, &parsed) == nil {
|
||||
upstream = parsed
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(perr.status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{ // headers already committed; nothing useful to do on encode failure
|
||||
@@ -655,7 +779,7 @@ func writeProviderUpstreamError(w http.ResponseWriter, providerName string, perr
|
||||
"message": providerName + " upstream error",
|
||||
"type": "upstream_error",
|
||||
"code": strconv.Itoa(perr.status),
|
||||
"upstream": upstream,
|
||||
"upstream": decodeUpstreamErrorBody(perr),
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -663,9 +787,10 @@ func writeProviderUpstreamError(w http.ResponseWriter, providerName string, perr
|
||||
// cacheCaptureWriter tees a cacheable request's response into an
|
||||
// in-memory buffer, capped at maxBodyBytes regardless of how large the
|
||||
// real response turns out to be, while writing everything through to the
|
||||
// wrapped *statusTrackingWriter unchanged — the mechanism runUnified uses
|
||||
// to fill the response cache (cache.go) on a miss without any adapter, or
|
||||
// forwardJSON/forwardStream inside one, needing to know caching exists.
|
||||
// wrapped *statusTrackingWriter unchanged — the mechanism runMeteredCall
|
||||
// uses to fill the response cache (cache.go) on a miss without any
|
||||
// adapter, or forwardJSON/forwardStream inside one, needing to know
|
||||
// caching exists.
|
||||
//
|
||||
// The cap matters even though the adapter response it captures is itself
|
||||
// already bounded by maxResponseBytes (32MiB): cache.maxBodyBytes
|
||||
@@ -681,9 +806,10 @@ func writeProviderUpstreamError(w http.ResponseWriter, providerName string, perr
|
||||
// overriding here. This is what "preserve statusTrackingWriter semantics"
|
||||
// means in practice: Flush passes through for free, and wroteHeader stays
|
||||
// the single shared bookkeeping field statusTrackingWriter already
|
||||
// maintains (handleAdapterError inspects it via the original sw pointer,
|
||||
// not through this wrapper, so a cacheCaptureWriter's WriteHeader/Write
|
||||
// must delegate to the embedded pointer, never shadow its state).
|
||||
// maintains (handleAdapterErrorEnvelope inspects it via the original sw
|
||||
// pointer, not through this wrapper, so a cacheCaptureWriter's
|
||||
// WriteHeader/Write must delegate to the embedded pointer, never shadow
|
||||
// its state).
|
||||
type cacheCaptureWriter struct {
|
||||
*statusTrackingWriter
|
||||
contentType string
|
||||
@@ -692,7 +818,7 @@ type cacheCaptureWriter struct {
|
||||
status int
|
||||
// oversize is true once buf has reached maxBodyBytes: Write stops
|
||||
// appending to buf from that point on (the real write to the client
|
||||
// is never affected), and runUnified skips calling store() entirely
|
||||
// is never affected), and runMeteredCall skips calling store() entirely
|
||||
// rather than handing it a silently truncated body.
|
||||
oversize bool
|
||||
}
|
||||
|
||||
+215
-1
@@ -283,6 +283,31 @@ func run() error {
|
||||
_, _ = w.Write([]byte(`"}]}}`))
|
||||
}))
|
||||
defer mcpProbeUpstream.Close()
|
||||
// anthropicProbeUpstream answers an Anthropic Messages API call with a
|
||||
// real Anthropic-shaped body, so the /v1/messages passthrough branch
|
||||
// (callAnthropicMessagesPassthrough) runs interpreted. A body
|
||||
// containing "make-it-fail" gets a 429, driving
|
||||
// handleAdapterErrorEnvelope's *providerHTTPError branch interpreted
|
||||
// too — the exact construct class that has broken under Yaegi before
|
||||
// (see handleAdapterError's own doc comment, routes_unified.go).
|
||||
anthropicProbeUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
raw, _ := io.ReadAll(r.Body)
|
||||
if bytes.Contains(raw, []byte("make-it-fail")) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"rate_limit_error","message":"slow down"}}`))
|
||||
return
|
||||
}
|
||||
if r.Header.Get("anthropic-version") == "" || r.Header.Get("x-api-key") == "" {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"type":"error","error":{"type":"invalid_request_error","message":"missing anthropic headers"}}`))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(`{"id":"msg_probe","type":"message","role":"assistant","model":"claude-upstream-real","content":[{"type":"text","text":"pong"}],"stop_reason":"end_turn","usage":{"input_tokens":7,"output_tokens":3}}`))
|
||||
}))
|
||||
defer anthropicProbeUpstream.Close()
|
||||
|
||||
// builtinLookupContextTokens (review fix, SHOULD-6): read directly
|
||||
// out of the generated pricing_data_gen.go rather than hardcoding a
|
||||
@@ -324,7 +349,12 @@ func run() error {
|
||||
attemptAccountingOverride := `{"breaker":{"failureThreshold":2,"openDuration":"3s","maxOpenDuration":"6s"},` +
|
||||
`"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},` +
|
||||
`"brk":{"type":"openai","baseUrl":"` + brkUpstream.URL + `","apiKey":"sk-up","discovery":true,"discoveryInterval":"1ms"},"` +
|
||||
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
|
||||
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]},` +
|
||||
// "anthropic" backs the /v1/messages passthrough probes
|
||||
// (exerciseMessagesRoute, below): a real anthropic-type provider,
|
||||
// interpreted end to end through
|
||||
// callAnthropicMessagesPassthrough.
|
||||
`"anthropic":{"type":"anthropic","baseUrl":"` + anthropicProbeUpstream.URL + `","apiKey":"sk-anth","models":["claude-test"]}},` + // #nosec G101 -- test fixture literal, not a real credential
|
||||
// modelMeta (feature v0.23): a config-override entry for
|
||||
// testDataWantModel, so exerciseHandler's GET /v1/models
|
||||
// assertion below proves resolveModelMeta's config-override
|
||||
@@ -680,6 +710,10 @@ func exerciseHandler(handler http.Handler, builtinContextTokens int) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := exerciseMessagesRoute(handler); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
unauthedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
||||
unauthedRec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(unauthedRec, unauthedReq)
|
||||
@@ -749,6 +783,186 @@ func exerciseStampedVersion(handler http.Handler) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// postMessages drives one POST /v1/messages through handler with the given
|
||||
// auth header name/value and JSON body, returning the recorder.
|
||||
func postMessages(handler http.Handler, hdrName, hdrValue, body string) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set(hdrName, hdrValue)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// exerciseMessagesRoute drives the inbound Anthropic Messages API route
|
||||
// under the REAL interpreter: both auth header forms, the streaming
|
||||
// rejection, a bad key's error envelope, an unknown model's error
|
||||
// envelope, the openai-type TRANSLATED path (openAIRequestFromAnthropic
|
||||
// -> chatCompletion -> anthropicResponseFromOpenAI), the anthropic-type
|
||||
// PASSTHROUGH path including its own upstream-error branch, and a
|
||||
// tool_use round trip through the translated path. This exists because
|
||||
// go test never runs under Yaegi — a construct that compiles and passes
|
||||
// `go test` can still panic or silently misbehave interpreted (this
|
||||
// package's own doc comment has the general case; handleAdapterError's
|
||||
// doc comment, routes_unified.go, has this route's own specific history
|
||||
// with plain type assertions over errors.As).
|
||||
func exerciseMessagesRoute(handler http.Handler) error {
|
||||
// 1. x-api-key auth + translated path (openai-type provider).
|
||||
rec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"`+testDataWantModel+`","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":[{"type":"text","text":"ping"}]}]}`)
|
||||
if rec.Code != http.StatusOK {
|
||||
return fmt.Errorf("POST /v1/messages (x-api-key, translated): status = %d, want 200, body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var msg map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &msg); err != nil {
|
||||
return fmt.Errorf("POST /v1/messages: decode body: %w (body=%s)", err, rec.Body.String())
|
||||
}
|
||||
if msg["type"] != "message" || msg["role"] != "assistant" {
|
||||
return fmt.Errorf("POST /v1/messages translated: type/role = %v/%v, want message/assistant: %s", msg["type"], msg["role"], rec.Body.String())
|
||||
}
|
||||
if msg["model"] != testDataWantModel {
|
||||
return fmt.Errorf("POST /v1/messages translated: model = %v, want %q (the client's requested alias echoed back, item 4 fix): %s", msg["model"], testDataWantModel, rec.Body.String())
|
||||
}
|
||||
if msg["stop_reason"] != "end_turn" {
|
||||
return fmt.Errorf("POST /v1/messages translated: stop_reason = %v, want end_turn: %s", msg["stop_reason"], rec.Body.String())
|
||||
}
|
||||
content, _ := msg["content"].([]any)
|
||||
if len(content) != 1 {
|
||||
return fmt.Errorf("POST /v1/messages translated: content len = %d, want 1: %s", len(content), rec.Body.String())
|
||||
}
|
||||
block, _ := content[0].(map[string]any)
|
||||
if block["type"] != "text" || block["text"] != "hi" {
|
||||
return fmt.Errorf("POST /v1/messages translated: content[0] = %v, want text/hi: %s", content[0], rec.Body.String())
|
||||
}
|
||||
usg, _ := msg["usage"].(map[string]any)
|
||||
if usg == nil || usg["input_tokens"] != float64(1) || usg["output_tokens"] != float64(1) {
|
||||
return fmt.Errorf("POST /v1/messages translated: usage = %v, want input=1 output=1: %s", msg["usage"], rec.Body.String())
|
||||
}
|
||||
|
||||
// 2. Authorization: Bearer must authenticate the same route.
|
||||
bearerRec := postMessages(handler, "Authorization", "Bearer "+testDataUserAPIKey,
|
||||
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
||||
if bearerRec.Code != http.StatusOK {
|
||||
return fmt.Errorf("POST /v1/messages (Authorization: Bearer): status = %d, want 200, body=%s", bearerRec.Code, bearerRec.Body.String())
|
||||
}
|
||||
|
||||
// 3. A bad key gets a 401 in the ANTHROPIC error shape.
|
||||
badRec := postMessages(handler, "x-api-key", "not-a-real-key",
|
||||
`{"model":"`+testDataWantModel+`","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
||||
if badRec.Code != http.StatusUnauthorized {
|
||||
return fmt.Errorf("POST /v1/messages with a bad key: status = %d, want 401, body=%s", badRec.Code, badRec.Body.String())
|
||||
}
|
||||
if err := assertAnthropicErrorShape("bad key 401", badRec.Body.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 4. "stream": true is rejected explicitly, in the Anthropic shape,
|
||||
// as a 400 (item 2 fix, 2026-08-22 review) — NOT 501: a 501 makes
|
||||
// anthropic-sdk-python (which retries any status >= 500) turn every
|
||||
// streaming call into a three-request retry storm against this
|
||||
// route's own rate limit. 400 maps to the SDK's non-retryable
|
||||
// BadRequestError.
|
||||
streamRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"`+testDataWantModel+`","max_tokens":64,"stream":true,"messages":[{"role":"user","content":"ping"}]}`)
|
||||
if streamRec.Code != http.StatusBadRequest {
|
||||
return fmt.Errorf("POST /v1/messages with stream=true: status = %d, want 400, body=%s", streamRec.Code, streamRec.Body.String())
|
||||
}
|
||||
if err := assertAnthropicErrorShape("stream rejection", streamRec.Body.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 5. An unknown model's 404 must also be Anthropic-shaped.
|
||||
unknownRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"no-such-model-anywhere","max_tokens":64,"messages":[{"role":"user","content":"ping"}]}`)
|
||||
if unknownRec.Code != http.StatusNotFound {
|
||||
return fmt.Errorf("POST /v1/messages with an unknown model: status = %d, want 404, body=%s", unknownRec.Code, unknownRec.Body.String())
|
||||
}
|
||||
if err := assertAnthropicErrorShape("unknown model 404", unknownRec.Body.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 6. The anthropic-type PASSTHROUGH branch, interpreted: every field
|
||||
// survives except "model", rewritten to the client's own requested
|
||||
// alias (item 4 fix) — a routing rewrite, not message-body
|
||||
// translation.
|
||||
passRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"claude-test","max_tokens":64,"system":"be terse","messages":[{"role":"user","content":"ping"}]}`)
|
||||
if passRec.Code != http.StatusOK {
|
||||
return fmt.Errorf("POST /v1/messages (anthropic passthrough): status = %d, want 200, body=%s", passRec.Code, passRec.Body.String())
|
||||
}
|
||||
var pass map[string]any
|
||||
if err := json.Unmarshal(passRec.Body.Bytes(), &pass); err != nil {
|
||||
return fmt.Errorf("POST /v1/messages passthrough: decode body: %w (body=%s)", err, passRec.Body.String())
|
||||
}
|
||||
if pass["id"] != "msg_probe" {
|
||||
return fmt.Errorf("POST /v1/messages passthrough: id = %v, want msg_probe (every field but \"model\" must survive the passthrough unchanged): %s", pass["id"], passRec.Body.String())
|
||||
}
|
||||
if pass["model"] != "claude-test" {
|
||||
return fmt.Errorf("POST /v1/messages passthrough: model = %v, want \"claude-test\" (the client's requested alias echoed back, item 4 fix — not anthropicProbeUpstream's own \"claude-upstream-real\"): %s", pass["model"], passRec.Body.String())
|
||||
}
|
||||
|
||||
// 6b. handleAdapterErrorEnvelope's *providerHTTPError type assertion,
|
||||
// interpreted — the exact construct class that has broken under Yaegi
|
||||
// before (see handleAdapterError's own doc comment, routes_unified.go).
|
||||
upErrRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"claude-test","max_tokens":64,"messages":[{"role":"user","content":"make-it-fail"}]}`)
|
||||
if upErrRec.Code != http.StatusTooManyRequests {
|
||||
return fmt.Errorf("POST /v1/messages (upstream 429): status = %d, want 429, body=%s", upErrRec.Code, upErrRec.Body.String())
|
||||
}
|
||||
if err := assertAnthropicErrorShape("upstream 429", upErrRec.Body.Bytes()); err != nil {
|
||||
return err
|
||||
}
|
||||
// A stale X-Llmgw-Cache header on an error response was deliberately
|
||||
// NOT asserted here (F6 fix, 2026-08-23 review): this harness's own
|
||||
// fixture configures no cache/Redis, so g.cache is nil and the
|
||||
// header is never set on any response regardless of what this route
|
||||
// does — the assertion could not fail no matter what code ran,
|
||||
// confirmed by a mutation pass that deleted the header-clear line it
|
||||
// was meant to guard (handleAdapterErrorEnvelope, routes_unified.go)
|
||||
// with make yaegi-check still reporting OK. That behavior is already
|
||||
// covered where it can actually fail — a real cache fixture, go
|
||||
// test: TestHandleChat_CacheableRequest_UpstreamDown_502HasNoCacheHeader
|
||||
// (routes_unified_test.go).
|
||||
|
||||
// 7. A tool_use round trip through the TRANSLATED path, to drive
|
||||
// openAIToolCallFromAnthropic/openAIToolMessageFromAnthropic and
|
||||
// anthropicToolInputFromOpenAI interpreted.
|
||||
toolRec := postMessages(handler, "x-api-key", testDataUserAPIKey,
|
||||
`{"model":"`+testDataWantModel+`","max_tokens":64,`+
|
||||
`"tools":[{"name":"get_weather","description":"w","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}],`+
|
||||
`"tool_choice":{"type":"auto"},`+
|
||||
`"messages":[{"role":"user","content":"weather?"},`+
|
||||
`{"role":"assistant","content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"London"}}]},`+
|
||||
`{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_1","content":"sunny"}]}]}`)
|
||||
if toolRec.Code != http.StatusOK {
|
||||
return fmt.Errorf("POST /v1/messages (tool_use round trip): status = %d, want 200, body=%s", toolRec.Code, toolRec.Body.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// assertAnthropicErrorShape checks b is {"type":"error","error":{"type":...,
|
||||
// "message":...}} — what an Anthropic SDK client parses.
|
||||
func assertAnthropicErrorShape(label string, b []byte) error {
|
||||
var env map[string]any
|
||||
if err := json.Unmarshal(b, &env); err != nil {
|
||||
return fmt.Errorf("/v1/messages %s: body is not JSON: %w (body=%s)", label, err, string(b))
|
||||
}
|
||||
if env["type"] != "error" {
|
||||
return fmt.Errorf("/v1/messages %s: top-level \"type\" = %v, want \"error\" (Anthropic envelope): %s", label, env["type"], string(b))
|
||||
}
|
||||
inner, _ := env["error"].(map[string]any)
|
||||
if inner == nil {
|
||||
return fmt.Errorf("/v1/messages %s: no \"error\" object: %s", label, string(b))
|
||||
}
|
||||
if _, ok := inner["type"].(string); !ok {
|
||||
return fmt.Errorf("/v1/messages %s: error.type missing: %s", label, string(b))
|
||||
}
|
||||
if _, ok := inner["message"].(string); !ok {
|
||||
return fmt.Errorf("/v1/messages %s: error.message missing: %s", label, string(b))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// exerciseFederatedTooLarge drives POST /mcp (federation's tools/call
|
||||
// path) against mcpProbeUpstream, whose response deliberately exceeds
|
||||
// mcpBackendCallResponseMaxBytes, and asserts the interpreted plugin
|
||||
|
||||
+549
-4
@@ -487,9 +487,34 @@ type anthropicContentBlock struct {
|
||||
|
||||
// anthropicUsagePayload is the "usage" object on both a non-streaming
|
||||
// Anthropic response and a streaming message_start/message_delta event.
|
||||
// CacheCreationInputTokens/CacheReadInputTokens (item 6 fix, 2026-08-22
|
||||
// review) are Anthropic's prompt-cache counters: tokens written to the
|
||||
// cache and tokens read back from it, reported separately from
|
||||
// InputTokens (fresh, uncached input). Verified repro before this fix:
|
||||
// 180000 cache-creation + 20000 cache-read tokens billed as tokin=4 — the
|
||||
// two cache fields were decoded into nothing and silently dropped.
|
||||
// Caching is near-universal for Claude Code and any agentic client using
|
||||
// Anthropic's prompt cache, so ignoring them under-counted budgets by
|
||||
// orders of magnitude on the primary use case.
|
||||
type anthropicUsagePayload struct {
|
||||
InputTokens int64 `json:"input_tokens"`
|
||||
OutputTokens int64 `json:"output_tokens"`
|
||||
InputTokens int64 `json:"input_tokens"`
|
||||
OutputTokens int64 `json:"output_tokens"`
|
||||
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens"`
|
||||
CacheReadInputTokens int64 `json:"cache_read_input_tokens"`
|
||||
}
|
||||
|
||||
// totalInputTokens returns the full input-side token count Anthropic
|
||||
// actually billed for one request: fresh InputTokens plus both cache-tier
|
||||
// counters. This gateway's budgets/rate-limits (limits.go's usage.prompt)
|
||||
// track one combined "prompt" token count per request, not per-tier
|
||||
// pricing, so folding all three into one number here — rather than
|
||||
// pricing cache-write/cache-read tiers differently — fixes the under-
|
||||
// billing item 6 reported (2026-08-22 review) without a materially larger
|
||||
// change to pricing.go's cost model, which this fix deliberately leaves
|
||||
// alone: token-budget accuracy was the reported defect, not $ cost
|
||||
// accuracy for cache-tier pricing.
|
||||
func (p anthropicUsagePayload) totalInputTokens() int64 {
|
||||
return p.InputTokens + p.CacheCreationInputTokens + p.CacheReadInputTokens
|
||||
}
|
||||
|
||||
// anthropicResponseBody is the subset of a non-streaming Anthropic
|
||||
@@ -559,7 +584,7 @@ func openAIResponseFromAnthropic(body []byte, model string, created int64) (map[
|
||||
message["tool_calls"] = toolCalls
|
||||
}
|
||||
|
||||
u := usage{prompt: resp.Usage.InputTokens, completion: resp.Usage.OutputTokens}
|
||||
u := usage{prompt: resp.Usage.totalInputTokens(), completion: resp.Usage.OutputTokens}
|
||||
|
||||
out := map[string]any{
|
||||
"id": chatCompletionIDPrefix + resp.ID,
|
||||
@@ -744,7 +769,7 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) {
|
||||
return nil, fmt.Errorf("%w: decode anthropic message_start event: %w", errUpstream, err)
|
||||
}
|
||||
st.id = p.Message.ID
|
||||
st.u.prompt = p.Message.Usage.InputTokens
|
||||
st.u.prompt = p.Message.Usage.totalInputTokens()
|
||||
return [][]byte{st.chunk(map[string]any{"role": "assistant"}, nil)}, nil
|
||||
|
||||
case "content_block_start":
|
||||
@@ -824,3 +849,523 @@ func (st *anthropicStreamState) translate(ev sseEvent) ([][]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// --- Anthropic -> OpenAI request translation (routes_messages.go) ---
|
||||
//
|
||||
// The functions below are the mirror of anthropicRequestFromOpenAI/
|
||||
// openAIResponseFromAnthropic above: routes_messages.go's /v1/messages
|
||||
// route receives an Anthropic-shaped request and, when it resolves to a
|
||||
// non-anthropic-type provider, must translate it to OpenAI's shape going
|
||||
// out and translate that provider's OpenAI-shaped response back to
|
||||
// Anthropic's shape coming back — the opposite direction from every
|
||||
// function above this comment, which exist to let an OpenAI-shaped
|
||||
// /v1/chat/completions request reach an anthropic-type provider. Neither
|
||||
// direction reuses the other's helpers beyond the truly shape-agnostic
|
||||
// ones (isTruthy, toFloat64/toInt64, systemTextFromContentParts): the two
|
||||
// wire formats are similar but not inverses of each other field-for-field
|
||||
// (Anthropic's tool_use.input is a decoded JSON value, OpenAI's
|
||||
// tool_calls[].function.arguments is that same value re-encoded as a
|
||||
// string, for instance), so each direction earns its own small mapping
|
||||
// functions rather than one trying to serve both.
|
||||
|
||||
// anthropicSystemToText maps Anthropic's "system" request field — a plain
|
||||
// string, or (newer API versions) an array of {"type":"text","text":...}
|
||||
// content blocks — to plain text. Reuses systemTextFromContentParts
|
||||
// (above) for the array form, since Anthropic's system content-block
|
||||
// shape and OpenAI's system-message content-array shape are identical.
|
||||
// Any other type (absent, or already-invalid JSON) returns "", nil —
|
||||
// openAIRequestFromAnthropic only emits a system message when this
|
||||
// returns non-empty text.
|
||||
func anthropicSystemToText(sys any) (string, error) {
|
||||
switch s := sys.(type) {
|
||||
case string:
|
||||
return s, nil
|
||||
case []any:
|
||||
return systemTextFromContentParts(s)
|
||||
default:
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
// openAIContentPartFromAnthropicImage maps one Anthropic {"type":"image",
|
||||
// "source":{"type":"base64","media_type":...,"data":...}} content block to
|
||||
// OpenAI's {"type":"image_url","image_url":{"url":"data:...;base64,..."}},
|
||||
// the mirror of anthropicImageSourceFromDataURI above. Anthropic's image
|
||||
// source "type" is currently always "base64" (a URL-sourced image source
|
||||
// is a separate, newer Anthropic feature this translator does not yet
|
||||
// support) — any other value is a *translateError.
|
||||
func openAIContentPartFromAnthropicImage(bm map[string]any) (map[string]any, error) {
|
||||
src, _ := bm["source"].(map[string]any)
|
||||
srcType, _ := src["type"].(string)
|
||||
if srcType != "base64" {
|
||||
return nil, &translateError{msg: fmt.Sprintf("image source type %q is not supported for this provider", srcType)}
|
||||
}
|
||||
mediaType, _ := src["media_type"].(string)
|
||||
data, _ := src["data"].(string)
|
||||
url := "data:" + mediaType + ";base64," + data
|
||||
return map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}}, nil
|
||||
}
|
||||
|
||||
// openAIToolCallFromAnthropic maps one Anthropic {"type":"tool_use","id",
|
||||
// "name","input"} content block to an OpenAI tool_calls[] entry
|
||||
// ({"id","type":"function","function":{"name","arguments"}}), the mirror
|
||||
// of anthropicToolUseBlockFromOpenAI above: input, already a decoded Go
|
||||
// value (this translator's caller decoded the whole request body into
|
||||
// map[string]any), is re-marshaled to a compact JSON string for OpenAI's
|
||||
// "arguments" field.
|
||||
func openAIToolCallFromAnthropic(bm map[string]any) (map[string]any, error) {
|
||||
id, _ := bm["id"].(string)
|
||||
name, _ := bm["name"].(string)
|
||||
input := bm["input"]
|
||||
if input == nil {
|
||||
input = map[string]any{}
|
||||
}
|
||||
argsBytes, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: marshal tool_use input: %w", errUpstream, err)
|
||||
}
|
||||
return map[string]any{
|
||||
"id": id,
|
||||
"type": "function",
|
||||
"function": map[string]any{
|
||||
"name": name,
|
||||
"arguments": string(argsBytes),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// openAIToolMessageFromAnthropic maps one Anthropic {"type":"tool_result",
|
||||
// "tool_use_id","content","is_error"} content block to an OpenAI
|
||||
// role:"tool" message ({"role":"tool","tool_call_id","content"}), the
|
||||
// mirror of anthropicToolResultBlock above. is_error (item 11/F4 fix,
|
||||
// 2026-08-22/23 review) has no native OpenAI equivalent — a tool message
|
||||
// carries no error flag — so it is preserved by prefixing "Error: " onto
|
||||
// content once openAIToolResultContentString (below) has normalized it
|
||||
// to a plain string, instead of being silently dropped: an earlier
|
||||
// version of this function read only tool_use_id/content and discarded
|
||||
// is_error entirely, and a second earlier version only prefixed a Go
|
||||
// string, still dropping the signal for Anthropic's array-content form.
|
||||
func openAIToolMessageFromAnthropic(bm map[string]any) map[string]any {
|
||||
toolUseID, _ := bm["tool_use_id"].(string)
|
||||
content := openAIToolResultContentString(bm["content"])
|
||||
if isErr, _ := bm["is_error"].(bool); isErr {
|
||||
content = "Error: " + content
|
||||
}
|
||||
return map[string]any{"role": "tool", "tool_call_id": toolUseID, "content": content}
|
||||
}
|
||||
|
||||
// openAIToolResultContentString normalizes an Anthropic tool_result
|
||||
// block's "content" into the single string OpenAI's tool message
|
||||
// "content" field requires (item 11/F4 fix, 2026-08-22/23 review). Three
|
||||
// source shapes all collapse to a string: a plain string passes through
|
||||
// unchanged; a content-block array (Anthropic permits this, and Claude
|
||||
// Code sends it for a tool_result carrying more than plain text) has its
|
||||
// text blocks concatenated; an absent/nil content becomes "". Handling
|
||||
// all three uniformly, here, is what lets openAIToolMessageFromAnthropic's
|
||||
// is_error prefix apply regardless of the source shape — the PREVIOUS
|
||||
// version only prefixed a Go string and left the array form unmarked
|
||||
// entirely, silently dropping the failure signal for real Claude Code
|
||||
// traffic (item 11 only covered half of it). It also fixes a second,
|
||||
// separate bug the array-only gap masked: an absent content previously
|
||||
// passed through as a literal Go nil, which this function's caller wrote
|
||||
// out as the JSON literal null — OpenAI rejects a null tool-message
|
||||
// content outright, where "" is accepted.
|
||||
func openAIToolResultContentString(content any) string {
|
||||
switch c := content.(type) {
|
||||
case string:
|
||||
return c
|
||||
case []any:
|
||||
var parts []string
|
||||
for _, block := range c {
|
||||
bm, ok := block.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if text, ok := bm["text"].(string); ok {
|
||||
parts = append(parts, text)
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, "")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// openAIMessagesFromAnthropic maps one Anthropic message ({"role",
|
||||
// "content"}, content either a plain string or a content-block array) to
|
||||
// one or more OpenAI messages, preserving the ORIGINAL block order (item
|
||||
// 3 fix, 2026-08-22 review): a plain string passes through unchanged. For
|
||||
// a block array, text/image/tool_use blocks accumulate into one pending
|
||||
// {role, content, tool_calls?} message; each tool_result block flushes
|
||||
// that pending message (if it has anything to flush) and then emits its
|
||||
// own role:"tool" message, in the SAME relative position the tool_result
|
||||
// held in the original array. Any trailing accumulated content flushes at
|
||||
// the end.
|
||||
//
|
||||
// This ordering matters beyond cosmetics: OpenAI requires a role:"tool"
|
||||
// message to directly follow the assistant message carrying the
|
||||
// tool_calls it answers, with nothing in between. The PREVIOUS version of
|
||||
// this function always emitted every tool_result AFTER the combined
|
||||
// content/tool_calls message regardless of where it actually appeared in
|
||||
// the original block array — correct for the common shape (a user turn
|
||||
// that is pure tool_results, following an assistant's pure tool_use
|
||||
// turn), but wrong for a legal Anthropic user turn shaped [tool_result,
|
||||
// text]: the old code emitted [user(text), tool(...)], inserting a
|
||||
// non-tool message between an assistant's tool_calls and its answer and
|
||||
// getting a 400 from OpenAI ("messages with role 'tool' must be a
|
||||
// response to a preceding message with 'tool_calls'"). Preserving
|
||||
// original order fixes both shapes without special-casing either.
|
||||
func openAIMessagesFromAnthropic(role string, content any) ([]any, error) {
|
||||
if s, ok := content.(string); ok {
|
||||
return []any{map[string]any{"role": role, "content": s}}, nil
|
||||
}
|
||||
|
||||
blocksRaw, ok := content.([]any)
|
||||
if !ok {
|
||||
// Absent or unrecognized content shape: an empty message rather
|
||||
// than an error — mirrors anthropicMessageFromOpenAI's own
|
||||
// tolerance of a missing "content" field above.
|
||||
return []any{map[string]any{"role": role, "content": ""}}, nil
|
||||
}
|
||||
|
||||
var out []any
|
||||
var pendingContent []any
|
||||
var pendingToolCalls []any
|
||||
flushPending := func() {
|
||||
if len(pendingContent) == 0 && len(pendingToolCalls) == 0 {
|
||||
return
|
||||
}
|
||||
m := map[string]any{"role": role}
|
||||
if len(pendingContent) > 0 {
|
||||
m["content"] = pendingContent
|
||||
} else {
|
||||
m["content"] = nil
|
||||
}
|
||||
if len(pendingToolCalls) > 0 {
|
||||
m["tool_calls"] = pendingToolCalls
|
||||
}
|
||||
out = append(out, m)
|
||||
pendingContent = nil
|
||||
pendingToolCalls = nil
|
||||
}
|
||||
|
||||
for _, b := range blocksRaw {
|
||||
bm, ok := b.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
switch bt, _ := bm["type"].(string); bt {
|
||||
case "text":
|
||||
text, _ := bm["text"].(string)
|
||||
pendingContent = append(pendingContent, map[string]any{"type": "text", "text": text})
|
||||
case "image":
|
||||
part, err := openAIContentPartFromAnthropicImage(bm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pendingContent = append(pendingContent, part)
|
||||
case "tool_use":
|
||||
tc, err := openAIToolCallFromAnthropic(bm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pendingToolCalls = append(pendingToolCalls, tc)
|
||||
case "tool_result":
|
||||
flushPending()
|
||||
out = append(out, openAIToolMessageFromAnthropic(bm))
|
||||
}
|
||||
}
|
||||
flushPending()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// openAIToolsFromAnthropic maps Anthropic's tools[]{name, description,
|
||||
// input_schema} to OpenAI's tools[].function{name, description,
|
||||
// parameters}, the mirror of anthropicToolsFromOpenAI above.
|
||||
func openAIToolsFromAnthropic(toolsRaw []any) []any {
|
||||
out := make([]any, 0, len(toolsRaw))
|
||||
for _, t := range toolsRaw {
|
||||
tm, ok := t.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fn := map[string]any{"name": tm["name"], "parameters": tm["input_schema"]}
|
||||
if desc, ok := tm["description"]; ok {
|
||||
fn["description"] = desc
|
||||
}
|
||||
out = append(out, map[string]any{"type": "function", "function": fn})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// openAIToolChoiceFromAnthropic maps Anthropic's tool_choice
|
||||
// ({"type":"auto"|"any"|"tool","name":...}) to OpenAI's tool_choice, the
|
||||
// mirror of anthropicToolChoiceFromOpenAI above. Anthropic has no
|
||||
// equivalent of OpenAI's "none" (a tool_choice this translator's forward
|
||||
// direction can only produce by omitting the field, per that function's
|
||||
// own doc comment), so there is no case here that ever needs to signal
|
||||
// "drop tools" the way that direction's dropTools return does.
|
||||
func openAIToolChoiceFromAnthropic(tc any) any {
|
||||
tm, ok := tc.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
switch t, _ := tm["type"].(string); t {
|
||||
case "auto":
|
||||
return "auto"
|
||||
case "any":
|
||||
return "required"
|
||||
case "tool":
|
||||
name, _ := tm["name"].(string)
|
||||
return map[string]any{"type": "function", "function": map[string]any{"name": name}}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// openAIReasoningModelPrefixes are OpenAI (and OpenAI-compatible) model
|
||||
// id prefixes that reject the "max_tokens" request field with a 400 and
|
||||
// require "max_completion_tokens" instead (item 10 fix, 2026-08-22
|
||||
// review): the o-series reasoning models and the gpt-5 family. Matched
|
||||
// against the resolved upstream model id, case-sensitively, the same
|
||||
// case OpenAI's own ids use.
|
||||
var openAIReasoningModelPrefixes = []string{"o1", "o3", "o4", "gpt-5"}
|
||||
|
||||
// isOpenAIReasoningModel reports whether model is one of the model
|
||||
// families openAIRequestFromAnthropic (below) must emit
|
||||
// "max_completion_tokens" for instead of "max_tokens" — see
|
||||
// openAIReasoningModelPrefixes' own doc comment. model may carry a
|
||||
// "provider/" prefix; this only needs a prefix match on the bare id
|
||||
// portion after it.
|
||||
func isOpenAIReasoningModel(model string) bool {
|
||||
if _, rest, ok := strings.Cut(model, "/"); ok {
|
||||
model = rest
|
||||
}
|
||||
for _, prefix := range openAIReasoningModelPrefixes {
|
||||
if strings.HasPrefix(model, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// openAIRequestFromAnthropic maps an Anthropic Messages API request body
|
||||
// (already decoded into map[string]any by the messages route's shared
|
||||
// admission/decode path) to an OpenAI chat-completion request body, the
|
||||
// mirror of anthropicRequestFromOpenAI above. It returns a
|
||||
// *translateError — never wrapped, matching this file's established
|
||||
// convention — for a content block this translator cannot map (an
|
||||
// unsupported image source type).
|
||||
//
|
||||
// The second return value lists request fields this translator
|
||||
// recognizes but has no OpenAI equivalent for and therefore dropped
|
||||
// (item 11 fix, 2026-08-22 review) — currently only "thinking"
|
||||
// (Anthropic's extended-thinking/reasoning-budget config). The caller
|
||||
// (routes_messages.go's callTranslatedMessages) logs a warning per
|
||||
// dropped field rather than this function logging directly: every other
|
||||
// function in this file is a pure, Gateway-free translator, callable and
|
||||
// testable without a *Gateway instance, and this keeps it that way.
|
||||
func openAIRequestFromAnthropic(req map[string]any) (map[string]any, []string, error) {
|
||||
out := map[string]any{}
|
||||
modelStr, _ := req["model"].(string)
|
||||
if modelStr != "" {
|
||||
out["model"] = modelStr
|
||||
}
|
||||
|
||||
msgs := make([]any, 0, 1)
|
||||
if sys, ok := req["system"]; ok {
|
||||
text, err := anthropicSystemToText(sys)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if text != "" {
|
||||
msgs = append(msgs, map[string]any{"role": "system", "content": text})
|
||||
}
|
||||
}
|
||||
|
||||
msgsRaw, _ := req["messages"].([]any)
|
||||
for _, m := range msgsRaw {
|
||||
msg, ok := m.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
role, _ := msg["role"].(string)
|
||||
converted, err := openAIMessagesFromAnthropic(role, msg["content"])
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
msgs = append(msgs, converted...)
|
||||
}
|
||||
out["messages"] = msgs
|
||||
|
||||
if v, ok := req["max_tokens"]; ok {
|
||||
if n, ok2 := toInt64(v); ok2 {
|
||||
// Reasoning models (o1/o3/o4/gpt-5 family) reject "max_tokens"
|
||||
// with a 400 and require "max_completion_tokens" instead
|
||||
// (item 10 fix, 2026-08-22 review) — isOpenAIReasoningModel's
|
||||
// own doc comment has the model-family list.
|
||||
if isOpenAIReasoningModel(modelStr) {
|
||||
out["max_completion_tokens"] = n
|
||||
} else {
|
||||
out["max_tokens"] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
if v, ok := req["temperature"]; ok {
|
||||
out["temperature"] = v
|
||||
}
|
||||
if v, ok := req["top_p"]; ok {
|
||||
out["top_p"] = v
|
||||
}
|
||||
if seqs, ok := req["stop_sequences"].([]any); ok && len(seqs) > 0 {
|
||||
out["stop"] = seqs
|
||||
}
|
||||
if v, ok := req["stream"]; ok {
|
||||
out["stream"] = v
|
||||
}
|
||||
|
||||
if toolsRaw, ok := req["tools"].([]any); ok {
|
||||
out["tools"] = openAIToolsFromAnthropic(toolsRaw)
|
||||
}
|
||||
if tc, ok := req["tool_choice"]; ok {
|
||||
if choice := openAIToolChoiceFromAnthropic(tc); choice != nil {
|
||||
out["tool_choice"] = choice
|
||||
}
|
||||
}
|
||||
|
||||
if meta, ok := req["metadata"].(map[string]any); ok {
|
||||
if uid, ok := meta["user_id"]; ok {
|
||||
out["user"] = uid
|
||||
}
|
||||
}
|
||||
|
||||
var dropped []string
|
||||
if _, ok := req["thinking"]; ok {
|
||||
dropped = append(dropped, "thinking")
|
||||
}
|
||||
|
||||
return out, dropped, nil
|
||||
}
|
||||
|
||||
// openAIChatResponseBody is the subset of a non-streaming OpenAI
|
||||
// chat.completion response this translator reads, the mirror of
|
||||
// anthropicResponseBody above. Message.Content is a pointer so a
|
||||
// tool-calls-only response (content: null) is told apart from one
|
||||
// carrying empty text — mirroring openAIResponseFromAnthropic's own
|
||||
// len(textParts) > 0 check in the other direction.
|
||||
type openAIChatResponseBody struct {
|
||||
ID string `json:"id"`
|
||||
Choices []struct {
|
||||
FinishReason string `json:"finish_reason"`
|
||||
Message struct {
|
||||
Content *string `json:"content"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
Function struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
} `json:"function"`
|
||||
} `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage chatStreamUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// anthropicToolInputFromOpenAI decodes an OpenAI tool_calls[].function.
|
||||
// arguments JSON string into a generic Go value for Anthropic's
|
||||
// tool_use.input field, the mirror of reMarshalToolInput above (which
|
||||
// goes the other way: a decoded value re-marshaled to a string). An empty
|
||||
// arguments string decodes to an empty object, matching a tool call with
|
||||
// no arguments.
|
||||
func anthropicToolInputFromOpenAI(argsStr string) (any, error) {
|
||||
input := any(map[string]any{})
|
||||
if argsStr != "" {
|
||||
if err := json.Unmarshal([]byte(argsStr), &input); err != nil {
|
||||
return nil, fmt.Errorf("%w: decode tool_calls[].function.arguments: %w", errUpstream, err)
|
||||
}
|
||||
}
|
||||
return input, nil
|
||||
}
|
||||
|
||||
// anthropicStopReasonFromOpenAI maps an OpenAI finish_reason to an
|
||||
// Anthropic stop_reason, the mirror of anthropicFinishReason above:
|
||||
// length -> max_tokens, tool_calls -> tool_use, everything else
|
||||
// (including "stop") -> end_turn. OpenAI's finish_reason carries no
|
||||
// equivalent of Anthropic's "stop_sequence" (which stop string matched),
|
||||
// so this translator can never populate the response's stop_sequence
|
||||
// field — the same loss of fidelity anthropicFinishReason already
|
||||
// accepts by collapsing end_turn and stop_sequence into one OpenAI value.
|
||||
func anthropicStopReasonFromOpenAI(finishReason string) string {
|
||||
switch finishReason {
|
||||
case "length":
|
||||
return "max_tokens"
|
||||
case "tool_calls":
|
||||
return "tool_use"
|
||||
default:
|
||||
return "end_turn"
|
||||
}
|
||||
}
|
||||
|
||||
// anthropicMessageIDFromOpenAI derives an Anthropic-shaped message id
|
||||
// from an OpenAI chat.completion response id, stripping
|
||||
// chatCompletionIDPrefix ("chatcmpl-") when present and prefixing
|
||||
// "msg_" — Anthropic's own id convention — the mirror of
|
||||
// openAIResponseFromAnthropic's chatCompletionIDPrefix+resp.ID above.
|
||||
func anthropicMessageIDFromOpenAI(id string) string {
|
||||
return "msg_" + strings.TrimPrefix(id, chatCompletionIDPrefix)
|
||||
}
|
||||
|
||||
// anthropicResponseFromOpenAI maps a non-streaming OpenAI chat.completion
|
||||
// response body to an Anthropic Messages API response body, the mirror of
|
||||
// openAIResponseFromAnthropic above. model is supplied by the caller
|
||||
// (routes_messages.go's callTranslatedMessages) — the client's original
|
||||
// requested alias, echoed back into the response's "model" field the same
|
||||
// way every translating adapter's gatewayAliasKey mechanism already does
|
||||
// for the outbound direction (routes_unified.go's gatewayAliasKey doc
|
||||
// comment).
|
||||
func anthropicResponseFromOpenAI(body []byte, model string) (map[string]any, error) {
|
||||
var resp openAIChatResponseBody
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, fmt.Errorf("%w: decode openai response: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
var content []any
|
||||
stopReason := "end_turn"
|
||||
if len(resp.Choices) > 0 {
|
||||
choice := resp.Choices[0]
|
||||
if choice.Message.Content != nil && *choice.Message.Content != "" {
|
||||
content = append(content, map[string]any{"type": "text", "text": *choice.Message.Content})
|
||||
}
|
||||
for _, tc := range choice.Message.ToolCalls {
|
||||
input, err := anthropicToolInputFromOpenAI(tc.Function.Arguments)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
content = append(content, map[string]any{
|
||||
"type": "tool_use",
|
||||
"id": tc.ID,
|
||||
"name": tc.Function.Name,
|
||||
"input": input,
|
||||
})
|
||||
}
|
||||
stopReason = anthropicStopReasonFromOpenAI(choice.FinishReason)
|
||||
}
|
||||
if content == nil {
|
||||
content = []any{}
|
||||
}
|
||||
|
||||
out := map[string]any{
|
||||
"id": anthropicMessageIDFromOpenAI(resp.ID),
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": content,
|
||||
"stop_reason": stopReason,
|
||||
"stop_sequence": nil,
|
||||
"usage": map[string]any{
|
||||
"input_tokens": resp.Usage.PromptTokens,
|
||||
"output_tokens": resp.Usage.CompletionTokens,
|
||||
},
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -507,3 +507,352 @@ func TestToFloat64(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// --- openAIRequestFromAnthropic / anthropicResponseFromOpenAI: the
|
||||
// reverse-direction translators routes_messages.go uses (/v1/messages
|
||||
// resolved to a non-anthropic-type provider) ---
|
||||
|
||||
// TestOpenAIRequestFromAnthropic_ToolResultBecomesToolMessage proves an
|
||||
// assistant tool_use block becomes an OpenAI tool_calls[] entry (with its
|
||||
// decoded "input" re-marshaled to a JSON string, matching OpenAI's
|
||||
// arguments field) and a following user tool_result block becomes its
|
||||
// own separate role:"tool" message — the mirror of
|
||||
// anthropicRequestFromOpenAI's pendingToolResults collapse in the other
|
||||
// direction.
|
||||
func TestOpenAIRequestFromAnthropic_ToolResultBecomesToolMessage(t *testing.T) {
|
||||
req := map[string]any{
|
||||
"model": "claude-x",
|
||||
"messages": []any{
|
||||
map[string]any{"role": "assistant", "content": []any{
|
||||
map[string]any{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": map[string]any{"city": "London"}},
|
||||
}},
|
||||
map[string]any{"role": "user", "content": []any{
|
||||
map[string]any{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
out, _, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIRequestFromAnthropic: %v", err)
|
||||
}
|
||||
msgs, _ := out["messages"].([]any)
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("messages = %v, want 2 (assistant tool_calls + tool result)", msgs)
|
||||
}
|
||||
assistant, _ := msgs[0].(map[string]any)
|
||||
toolCalls, _ := assistant["tool_calls"].([]any)
|
||||
if len(toolCalls) != 1 {
|
||||
t.Fatalf("assistant tool_calls = %v, want 1 entry", assistant["tool_calls"])
|
||||
}
|
||||
tc, _ := toolCalls[0].(map[string]any)
|
||||
fn, _ := tc["function"].(map[string]any)
|
||||
assert.Equal(t, "get_weather", fn["name"])
|
||||
assert.JSONEq(t, `{"city":"London"}`, fn["arguments"].(string))
|
||||
|
||||
toolMsg, _ := msgs[1].(map[string]any)
|
||||
assert.Equal(t, "tool", toolMsg["role"])
|
||||
assert.Equal(t, "call_1", toolMsg["tool_call_id"])
|
||||
assert.Equal(t, "sunny", toolMsg["content"])
|
||||
}
|
||||
|
||||
// TestOpenAIRequestFromAnthropic_SystemStringFoldedIntoMessages proves a
|
||||
// plain-string Anthropic "system" field becomes a role:"system" message
|
||||
// prepended ahead of the rest, and the Anthropic-only top-level "system"
|
||||
// key never survives into the OpenAI-shaped output.
|
||||
func TestOpenAIRequestFromAnthropic_SystemStringFoldedIntoMessages(t *testing.T) {
|
||||
req := map[string]any{
|
||||
"model": "claude-x",
|
||||
"system": "Be terse.",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hi"}},
|
||||
}
|
||||
|
||||
out, _, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIRequestFromAnthropic: %v", err)
|
||||
}
|
||||
if _, ok := out["system"]; ok {
|
||||
t.Error(`out carries a "system" key; Anthropic's system field has no OpenAI equivalent and must be folded into messages[]`)
|
||||
}
|
||||
msgs, _ := out["messages"].([]any)
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("messages = %v, want 2 (system + user)", msgs)
|
||||
}
|
||||
sysMsg, _ := msgs[0].(map[string]any)
|
||||
assert.Equal(t, "system", sysMsg["role"])
|
||||
assert.Equal(t, "Be terse.", sysMsg["content"])
|
||||
}
|
||||
|
||||
// TestAnthropicResponseFromOpenAI_ToolCallsBecomeToolUseBlocks proves an
|
||||
// OpenAI tool_calls[] entry becomes an Anthropic tool_use content block
|
||||
// with its arguments STRING decoded back to a structured "input" value,
|
||||
// stop_reason maps tool_calls -> tool_use, and the caller-supplied model
|
||||
// (the client's own requested alias) is echoed into the response.
|
||||
func TestAnthropicResponseFromOpenAI_ToolCallsBecomeToolUseBlocks(t *testing.T) {
|
||||
const body = `{"id":"chatcmpl-77","choices":[{"index":0,"message":{"content":null,"tool_calls":[{"id":"call_9","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":3,"completion_tokens":2}}`
|
||||
|
||||
out, err := anthropicResponseFromOpenAI([]byte(body), "aliased/model")
|
||||
if err != nil {
|
||||
t.Fatalf("anthropicResponseFromOpenAI: %v", err)
|
||||
}
|
||||
assert.Equal(t, "aliased/model", out["model"])
|
||||
assert.Equal(t, "tool_use", out["stop_reason"])
|
||||
|
||||
content, _ := out["content"].([]any)
|
||||
if len(content) != 1 {
|
||||
t.Fatalf("content = %v, want one tool_use block", out["content"])
|
||||
}
|
||||
block, _ := content[0].(map[string]any)
|
||||
assert.Equal(t, "tool_use", block["type"])
|
||||
assert.Equal(t, "get_weather", block["name"])
|
||||
assert.Equal(t, "call_9", block["id"])
|
||||
input, _ := block["input"].(map[string]any)
|
||||
assert.Equal(t, "Paris", input["city"])
|
||||
|
||||
usageMap, _ := out["usage"].(map[string]any)
|
||||
assert.Equal(t, int64(3), usageMap["input_tokens"])
|
||||
assert.Equal(t, int64(2), usageMap["output_tokens"])
|
||||
}
|
||||
|
||||
// TestAnthropicResponseFromOpenAI_PlainTextResponse proves the common
|
||||
// case: a plain-text OpenAI response becomes a single Anthropic text
|
||||
// content block with stop_reason "stop" -> "end_turn".
|
||||
func TestAnthropicResponseFromOpenAI_PlainTextResponse(t *testing.T) {
|
||||
const body = `{"id":"chatcmpl-1","choices":[{"index":0,"message":{"content":"hi there"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":2}}`
|
||||
|
||||
out, err := anthropicResponseFromOpenAI([]byte(body), "claude-x")
|
||||
if err != nil {
|
||||
t.Fatalf("anthropicResponseFromOpenAI: %v", err)
|
||||
}
|
||||
assert.Equal(t, "message", out["type"])
|
||||
assert.Equal(t, "assistant", out["role"])
|
||||
assert.Equal(t, "end_turn", out["stop_reason"])
|
||||
content, _ := out["content"].([]any)
|
||||
if len(content) != 1 {
|
||||
t.Fatalf("content = %v, want one text block", out["content"])
|
||||
}
|
||||
block, _ := content[0].(map[string]any)
|
||||
assert.Equal(t, "text", block["type"])
|
||||
assert.Equal(t, "hi there", block["text"])
|
||||
}
|
||||
|
||||
// TestOpenAIContentPartFromAnthropicImage_RejectsNonBase64Source proves an
|
||||
// Anthropic image block whose source is not base64-encoded (a URL-sourced
|
||||
// image, which this translator does not support) returns a
|
||||
// *translateError rather than silently dropping or mistranslating it.
|
||||
func TestOpenAIContentPartFromAnthropicImage_RejectsNonBase64Source(t *testing.T) {
|
||||
_, err := openAIContentPartFromAnthropicImage(map[string]any{"source": map[string]any{"type": "url", "url": "https://example.com/x.png"}})
|
||||
if err == nil {
|
||||
t.Fatal("want an error for a non-base64 image source, got nil")
|
||||
}
|
||||
if _, ok := err.(*translateError); !ok {
|
||||
t.Errorf("err = %T, want *translateError", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIMessagesFromAnthropic_PreservesToolResultThenTextOrder is the
|
||||
// regression for item 3 (IMPORTANT, 2026-08-22 review): a legal Anthropic
|
||||
// user turn shaped [tool_result, text] must translate preserving that
|
||||
// order — tool message first, then the content message — since OpenAI
|
||||
// requires a role:"tool" message to directly follow the assistant
|
||||
// message carrying the tool_calls it answers, with nothing in between.
|
||||
// The previous version of this function always emitted every tool_result
|
||||
// AFTER the combined content/tool_calls message regardless of its
|
||||
// original position, which turned this exact shape into
|
||||
// [assistant(tool_calls), user(text), tool(...)] — an OpenAI 400, since
|
||||
// user(text) sits between the tool_calls and its answer.
|
||||
func TestOpenAIMessagesFromAnthropic_PreservesToolResultThenTextOrder(t *testing.T) {
|
||||
content := []any{
|
||||
map[string]any{"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"},
|
||||
map[string]any{"type": "text", "text": "thanks"},
|
||||
}
|
||||
|
||||
out, err := openAIMessagesFromAnthropic("user", content)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIMessagesFromAnthropic: %v", err)
|
||||
}
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("messages = %v, want 2 (tool result, then text)", out)
|
||||
}
|
||||
|
||||
first, _ := out[0].(map[string]any)
|
||||
if first["role"] != "tool" || first["tool_call_id"] != "call_1" {
|
||||
t.Errorf("messages[0] = %v, want the tool_result message FIRST, matching its original position", first)
|
||||
}
|
||||
|
||||
second, _ := out[1].(map[string]any)
|
||||
if second["role"] != "user" {
|
||||
t.Errorf("messages[1] = %v, want the text-content message SECOND", second)
|
||||
}
|
||||
parts, _ := second["content"].([]any)
|
||||
if len(parts) != 1 {
|
||||
t.Fatalf("messages[1].content = %v, want one text part", second["content"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIMessagesFromAnthropic_InterspersedToolResults proves the fix
|
||||
// generalizes beyond one tool_result: [tool_result A, text, tool_result
|
||||
// B] must translate to [tool(A), user(text), tool(B)] — each tool_result
|
||||
// flushing whatever content/tool_calls had accumulated before it, not
|
||||
// just the first one.
|
||||
func TestOpenAIMessagesFromAnthropic_InterspersedToolResults(t *testing.T) {
|
||||
content := []any{
|
||||
map[string]any{"type": "tool_result", "tool_use_id": "call_A", "content": "a"},
|
||||
map[string]any{"type": "text", "text": "middle"},
|
||||
map[string]any{"type": "tool_result", "tool_use_id": "call_B", "content": "b"},
|
||||
}
|
||||
|
||||
out, err := openAIMessagesFromAnthropic("user", content)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIMessagesFromAnthropic: %v", err)
|
||||
}
|
||||
if len(out) != 3 {
|
||||
t.Fatalf("messages = %v, want 3 (tool A, text, tool B)", out)
|
||||
}
|
||||
m0, _ := out[0].(map[string]any)
|
||||
m1, _ := out[1].(map[string]any)
|
||||
m2, _ := out[2].(map[string]any)
|
||||
if m0["tool_call_id"] != "call_A" {
|
||||
t.Errorf("messages[0].tool_call_id = %v, want call_A", m0["tool_call_id"])
|
||||
}
|
||||
if m1["role"] != "user" {
|
||||
t.Errorf("messages[1].role = %v, want user (the interspersed text)", m1["role"])
|
||||
}
|
||||
if m2["tool_call_id"] != "call_B" {
|
||||
t.Errorf("messages[2].tool_call_id = %v, want call_B", m2["tool_call_id"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIToolMessageFromAnthropic_PreservesIsError is the regression
|
||||
// for item 11a and F4 (IMPORTANT/BLOCKING-adjacent, 2026-08-22/23
|
||||
// review): Anthropic's tool_result.is_error has no native OpenAI
|
||||
// tool-message field, so a prior version of this function silently
|
||||
// dropped it — and a second prior version prefixed only a Go string,
|
||||
// silently dropping the signal again for the array-content form Claude
|
||||
// Code actually sends. All content shapes now normalize to a string
|
||||
// (openAIToolResultContentString) before the "Error: " prefix applies,
|
||||
// and an absent content becomes "" rather than the JSON null OpenAI
|
||||
// rejects for a tool message.
|
||||
func TestOpenAIToolMessageFromAnthropic_PreservesIsError(t *testing.T) {
|
||||
cases := []struct {
|
||||
content any
|
||||
wantContent any
|
||||
name string
|
||||
isError bool
|
||||
}{
|
||||
{name: "string content with is_error prefixes Error:", content: "boom", isError: true, wantContent: "Error: boom"},
|
||||
{name: "string content without is_error is unchanged", content: "ok", isError: false, wantContent: "ok"},
|
||||
{name: "array-form content with is_error is normalized to a string and prefixed (F4)", content: []any{map[string]any{"type": "text", "text": "boom"}}, isError: true, wantContent: "Error: boom"},
|
||||
{name: "array-form content without is_error joins its text blocks (F4)", content: []any{map[string]any{"type": "text", "text": "part1"}, map[string]any{"type": "text", "text": "part2"}}, isError: false, wantContent: "part1part2"},
|
||||
{name: "absent content with is_error still prefixes onto an empty string, never null (F4)", content: nil, isError: true, wantContent: "Error: "},
|
||||
{name: "absent content without is_error is an empty string, never null (F4)", content: nil, isError: false, wantContent: ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
bm := map[string]any{"tool_use_id": "call_1", "is_error": tc.isError}
|
||||
if tc.content != nil {
|
||||
bm["content"] = tc.content
|
||||
}
|
||||
got := openAIToolMessageFromAnthropic(bm)
|
||||
assert.Equal(t, tc.wantContent, got["content"])
|
||||
if _, ok := got["content"].(string); !ok {
|
||||
t.Errorf("content = %#v (%T), want a string (OpenAI rejects a null/non-string tool message content)", got["content"], got["content"])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIRequestFromAnthropic_ThinkingFieldReportedAsDropped is the
|
||||
// regression for item 11b (IMPORTANT, 2026-08-22 review): Anthropic's
|
||||
// "thinking" (extended-thinking config) has no OpenAI equivalent.
|
||||
// openAIRequestFromAnthropic must report it in its dropped-fields return
|
||||
// rather than silently discarding it, so the caller (routes_messages.go's
|
||||
// callTranslatedMessages) can log a warning instead of the drop being
|
||||
// invisible.
|
||||
func TestOpenAIRequestFromAnthropic_ThinkingFieldReportedAsDropped(t *testing.T) {
|
||||
req := map[string]any{
|
||||
"model": "claude-x",
|
||||
"thinking": map[string]any{"type": "enabled", "budget_tokens": float64(1024)},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hi"}},
|
||||
}
|
||||
|
||||
_, dropped, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIRequestFromAnthropic: %v", err)
|
||||
}
|
||||
if len(dropped) != 1 || dropped[0] != "thinking" {
|
||||
t.Errorf("dropped = %v, want [\"thinking\"]", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIRequestFromAnthropic_ThinkingAbsent_NothingDropped proves the
|
||||
// dropped-fields list stays empty for a request that never set
|
||||
// "thinking" in the first place — the previous test's absence is not
|
||||
// itself evidence of a bug.
|
||||
func TestOpenAIRequestFromAnthropic_ThinkingAbsent_NothingDropped(t *testing.T) {
|
||||
req := map[string]any{
|
||||
"model": "claude-x",
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hi"}},
|
||||
}
|
||||
|
||||
_, dropped, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIRequestFromAnthropic: %v", err)
|
||||
}
|
||||
if len(dropped) != 0 {
|
||||
t.Errorf("dropped = %v, want none", dropped)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenAIRequestFromAnthropic_ReasoningModel_UsesMaxCompletionTokens is
|
||||
// the regression for item 10 (IMPORTANT, 2026-08-22 review): OpenAI's
|
||||
// o-series and gpt-5-family reasoning models reject the "max_tokens"
|
||||
// request field with a 400 and require "max_completion_tokens" instead.
|
||||
// Every other model keeps using "max_tokens" as before.
|
||||
func TestOpenAIRequestFromAnthropic_ReasoningModel_UsesMaxCompletionTokens(t *testing.T) {
|
||||
cases := []struct {
|
||||
model string
|
||||
wantField string
|
||||
wantOthers []string
|
||||
}{
|
||||
{"o3-mini", "max_completion_tokens", []string{"max_tokens"}},
|
||||
{"o1", "max_completion_tokens", []string{"max_tokens"}},
|
||||
{"o4-mini-2025-04-16", "max_completion_tokens", []string{"max_tokens"}},
|
||||
{"gpt-5", "max_completion_tokens", []string{"max_tokens"}},
|
||||
{"gpt-5.1-chat-latest", "max_completion_tokens", []string{"max_tokens"}},
|
||||
{"gpt-4o", "max_tokens", []string{"max_completion_tokens"}},
|
||||
{"gpt-4o-mini", "max_tokens", []string{"max_completion_tokens"}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.model, func(t *testing.T) {
|
||||
req := map[string]any{
|
||||
"model": tc.model,
|
||||
"max_tokens": float64(256),
|
||||
"messages": []any{map[string]any{"role": "user", "content": "hi"}},
|
||||
}
|
||||
out, _, err := openAIRequestFromAnthropic(req)
|
||||
if err != nil {
|
||||
t.Fatalf("openAIRequestFromAnthropic: %v", err)
|
||||
}
|
||||
if _, ok := out[tc.wantField]; !ok {
|
||||
t.Errorf("out[%q] missing, want it set to 256", tc.wantField)
|
||||
}
|
||||
for _, other := range tc.wantOthers {
|
||||
if _, ok := out[other]; ok {
|
||||
t.Errorf("out[%q] present, want only %q set", other, tc.wantField)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnthropicUsagePayload_TotalInputTokens_FoldsCacheCounters is the
|
||||
// unit-level regression for item 6 (IMPORTANT, 2026-08-22 review):
|
||||
// cache_creation_input_tokens and cache_read_input_tokens must fold into
|
||||
// the billed prompt count alongside fresh input_tokens, not be dropped.
|
||||
func TestAnthropicUsagePayload_TotalInputTokens_FoldsCacheCounters(t *testing.T) {
|
||||
p := anthropicUsagePayload{InputTokens: 4, CacheCreationInputTokens: 180000, CacheReadInputTokens: 20000}
|
||||
if got, want := p.totalInputTokens(), int64(200004); got != want {
|
||||
t.Errorf("totalInputTokens() = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user