mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-14 05:18:54 +00:00
feat: unified chat and embeddings endpoints with limits and accounting
Implements Task 12: POST /v1/chat/completions and POST /v1/embeddings
through a shared pipeline (routes_unified.go: handleChat, handleEmbeddings,
runUnified) wired into ServeHTTP's dispatch in llmgateway.go.
Pipeline: decode (10MiB cap) -> resolve model -> enforce user/group
limits -> call adapter -> account usage, translating every failure mode
into its HTTP envelope per the task brief's carried rulings:
- ALIAS ECHO (ruling a): the client-requested model id is threaded to the
adapter via the internal gatewayAliasKey convention and echoed back in
anthropic/gemini's translated response model field; every adapter
strips the key before building its upstream wire body so it never
leaks to a real provider.
- Limit violations (ruling b): storeDown -> 503 server_error; otherwise
429 rate_limit_error with Retry-After when the violation names a
window.
- Stream-safe error handling (ruling c): statusTrackingWriter.wroteHeader
gates whether an adapter error still gets an envelope; providerHTTPError
wraps into {error:{message,type,code,upstream}}; translateError maps to
400/501; context.Canceled logs quietly; anything else is a 502.
- Usage/accounting (ruling d): zero non-streaming usage gets a
ceil(len(body)/4) prompt-token estimate; zero streaming usage logs once
and accounts the request only. unifiedCostMicros extends costMicros'
single-id lookup to canonical-then-bare without changing its signature.
- Scopes (ruling e): user scope only when u.limits is set, group scope
only when grp.limits is set, user first.
routes_unified_test.go drives every case end-to-end through
Gateway.ServeHTTP against httptest fake upstreams (openai- and
anthropic-shaped): happy path with counters asserted via the limiter's
own store, incremental stream flushing, 401/403/404/429/503, embeddings,
alias echo through a provider-prefixed anthropic request, wrapped
upstream error envelopes, and a mid-stream drop that must not append a
second envelope to an already-committed response.
Gates: gofmt, go vet, fieldalignment, golangci-lint (incl. gosec) all
clean; go test ./... -race -count=1 passes.
This commit is contained in:
+20
-3
@@ -84,6 +84,16 @@ func geminiModelFromRequest(req map[string]any) string {
|
||||
// request is made.
|
||||
func (a *geminiAdapter) chatCompletion(ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error) {
|
||||
model := geminiModelFromRequest(req)
|
||||
// responseModel (ruling a, ALIAS ECHO): gatewayAliasKey, when present,
|
||||
// is the exact id the client requested — echoed into the translated
|
||||
// response's "model" field, distinct from model (the bare upstream id),
|
||||
// which must keep driving the URL below unchanged. Deleted from req so
|
||||
// it cannot leak into the upstream request body.
|
||||
responseModel := model
|
||||
if alias, ok := req[gatewayAliasKey].(string); ok && alias != "" {
|
||||
responseModel = alias
|
||||
}
|
||||
delete(req, gatewayAliasKey)
|
||||
streaming, _ := req["stream"].(bool)
|
||||
|
||||
body, err := geminiRequestFromOpenAI(req)
|
||||
@@ -117,11 +127,11 @@ func (a *geminiAdapter) chatCompletion(ctx context.Context, w http.ResponseWrite
|
||||
}
|
||||
|
||||
if streaming && strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
|
||||
return a.forwardStream(w, resp.Body, model)
|
||||
return a.forwardStream(w, resp.Body, responseModel)
|
||||
}
|
||||
// Either a non-streaming request, or a streaming one whose upstream
|
||||
// ignored stream=true and answered with a normal JSON body.
|
||||
return a.forwardJSON(w, resp, model)
|
||||
return a.forwardJSON(w, resp, responseModel)
|
||||
}
|
||||
|
||||
// forwardJSON reads resp's full body (capped at maxResponseBytes),
|
||||
@@ -189,6 +199,13 @@ func (a *geminiAdapter) forwardStream(w http.ResponseWriter, body io.Reader, gat
|
||||
// Embeddings never stream.
|
||||
func (a *geminiAdapter) embeddings(ctx context.Context, w http.ResponseWriter, req map[string]any) (usage, error) {
|
||||
model := geminiModelFromRequest(req)
|
||||
// responseModel (ruling a, ALIAS ECHO): see chatCompletion's identical
|
||||
// pattern above.
|
||||
responseModel := model
|
||||
if alias, ok := req[gatewayAliasKey].(string); ok && alias != "" {
|
||||
responseModel = alias
|
||||
}
|
||||
delete(req, gatewayAliasKey)
|
||||
|
||||
texts, single, err := geminiEmbeddingInputTexts(req["input"])
|
||||
if err != nil {
|
||||
@@ -223,7 +240,7 @@ func (a *geminiAdapter) embeddings(ctx context.Context, w http.ResponseWriter, r
|
||||
return usage{}, fmt.Errorf("%w: read response body: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
out, u, err := openAIEmbeddingResponseFromGemini(raw, single, model, estimatedEmbeddingTokens(texts))
|
||||
out, u, err := openAIEmbeddingResponseFromGemini(raw, single, responseModel, estimatedEmbeddingTokens(texts))
|
||||
if err != nil {
|
||||
return usage{}, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user