mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
feat(models): emit the context window under three client dialects
OpenAI's /v1/models schema carries no context field, so every client that wants one reads a vendor dialect, and a client whose dialect is absent does not ask — it substitutes its own default. GET /v1/models now emits the same resolved integer under "context_window", "context_length" (the OpenRouter spelling) and "max_model_len" (the vLLM spelling). All three are omitted together when the window is unknown, exactly as the single field was. Measured case: omp 18.1.15's openai-models-list discovery resolves `max_model_len ?? context_length ?? <bundled catalog> ?? 128000` and never reads context_window, so a 1048576-context local model reached the client as 128000, which also clamped its maxTokens. The metadata fetcher already parses all of these dialects on the inbound side (provider_openai.go), so this makes the outbound direction speak what the inbound one already understands. Claude-Session: https://claude.ai/code/session_0168ukk3uwp8PgiuRAzeLPXp
This commit is contained in:
@@ -666,8 +666,8 @@ identical file with nothing to review.
|
|||||||
|
|
||||||
### `GET /v1/models` extension fields
|
### `GET /v1/models` extension fields
|
||||||
|
|
||||||
Two OpenAI-compat-safe extra fields on each model object, present only
|
OpenAI-compat-safe extra fields on each model object, present only when
|
||||||
when known:
|
known:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -675,11 +675,19 @@ when known:
|
|||||||
"object": "model",
|
"object": "model",
|
||||||
"owned_by": "openai",
|
"owned_by": "openai",
|
||||||
"context_window": 400000,
|
"context_window": 400000,
|
||||||
|
"context_length": 400000,
|
||||||
|
"max_model_len": 400000,
|
||||||
"pricing": {"input_per_mtok_usd": 1.25, "output_per_mtok_usd": 10}
|
"pricing": {"input_per_mtok_usd": 1.25, "output_per_mtok_usd": 10}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
- `context_window` — an integer token count, omitted when unknown.
|
- `context_window`, `context_length`, `max_model_len` — the same integer
|
||||||
|
token count under all three dialect spellings, omitted together when
|
||||||
|
unknown. OpenAI's own schema has no context field, so each client reads
|
||||||
|
whichever vendor dialect it was built against: `context_length` is the
|
||||||
|
OpenRouter spelling, `max_model_len` the vLLM one. A client whose
|
||||||
|
dialect is missing does not ask — it silently substitutes its own
|
||||||
|
default (128k is a common one), so all three are emitted.
|
||||||
- `pricing` — `{input_per_mtok_usd, output_per_mtok_usd}`, both floats
|
- `pricing` — `{input_per_mtok_usd, output_per_mtok_usd}`, both floats
|
||||||
in USD per 1,000,000 tokens. Omitted when unknown; present with `0`
|
in USD per 1,000,000 tokens. Omitted when unknown; present with `0`
|
||||||
values for an explicitly free model — a known zero, not an absent
|
values for an explicitly free model — a known zero, not an absent
|
||||||
|
|||||||
+19
-7
@@ -1563,17 +1563,29 @@ func (m *modelRegistry) aliasEntriesFor(grp *group) map[string]map[string]any {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// modelObject builds one OpenAI-compatible model list entry, extended
|
// modelObject builds one OpenAI-compatible model list entry, extended
|
||||||
// with two OpenAI-compat-safe extension fields (feature v0.23):
|
// with OpenAI-compat-safe extension fields (feature v0.23): the context
|
||||||
// "context_window" (int) and "pricing" ({"input_per_mtok_usd",
|
// window and "pricing" ({"input_per_mtok_usd", "output_per_mtok_usd"}
|
||||||
// "output_per_mtok_usd"} floats, USD per million tokens) — both omitted
|
// floats, USD per million tokens) — both omitted entirely when meta
|
||||||
// entirely when meta reports them unknown, never emitted as a
|
// reports them unknown, never emitted as a misleading zero. "pricing" IS
|
||||||
// misleading zero. "pricing" IS emitted with zeros for an explicitly
|
// emitted with zeros for an explicitly free model (meta.CostKnown true,
|
||||||
// free model (meta.CostKnown true, both cost fields 0) — that is a
|
// both cost fields 0) — that is a known, meaningful zero, not an absent
|
||||||
// known, meaningful zero, not an absent one.
|
// one.
|
||||||
|
//
|
||||||
|
// The context window is emitted under THREE keys carrying the identical
|
||||||
|
// integer: "context_window", "context_length" (the OpenRouter spelling)
|
||||||
|
// and "max_model_len" (the vLLM spelling). OpenAI's own /v1/models
|
||||||
|
// schema has no context field at all, so every client that wants one
|
||||||
|
// reads some vendor dialect, and a client whose dialect is missing
|
||||||
|
// silently substitutes its own default instead of asking. Emitting all
|
||||||
|
// three costs two map entries and removes that whole failure class —
|
||||||
|
// the same reason the metadata FETCHER already parses all of these
|
||||||
|
// dialects on the inbound side (provider_openai.go's modelMetadata).
|
||||||
func modelObject(id, ownedBy string, meta resolvedModelMeta) map[string]any {
|
func modelObject(id, ownedBy string, meta resolvedModelMeta) map[string]any {
|
||||||
obj := map[string]any{"id": id, "object": "model", "owned_by": ownedBy}
|
obj := map[string]any{"id": id, "object": "model", "owned_by": ownedBy}
|
||||||
if meta.ContextKnown {
|
if meta.ContextKnown {
|
||||||
obj["context_window"] = meta.ContextTokens
|
obj["context_window"] = meta.ContextTokens
|
||||||
|
obj["context_length"] = meta.ContextTokens
|
||||||
|
obj["max_model_len"] = meta.ContextTokens
|
||||||
}
|
}
|
||||||
if meta.CostKnown {
|
if meta.CostKnown {
|
||||||
obj["pricing"] = map[string]any{
|
obj["pricing"] = map[string]any{
|
||||||
|
|||||||
@@ -375,3 +375,36 @@ func TestModelRegistry_RefreshProvider_MetadataPanicDoesNotFailDiscovery(t *test
|
|||||||
t.Error("want at least one log line recording the metadata-capture panic")
|
t.Error("want at least one log line recording the metadata-capture panic")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestModelObject_ContextDialectMirrors proves every model object carries its
|
||||||
|
// context window under all three dialect keys OpenAI-compatible clients
|
||||||
|
// actually read — "context_window", "context_length" and "max_model_len" —
|
||||||
|
// always the identical integer, and that all three vanish together when the
|
||||||
|
// window is unknown.
|
||||||
|
//
|
||||||
|
// Client-side reason this exists (2026-09-10): omp's openai-models-list
|
||||||
|
// discovery reads `max_model_len ?? context_length ?? <bundled catalog> ??
|
||||||
|
// 128000` and never looks at context_window, so a 1M-context local model was
|
||||||
|
// advertised to it as the 128000 default. Other clients read the OpenRouter
|
||||||
|
// spelling (context_length). One resolved value, three spellings, no
|
||||||
|
// per-client configuration.
|
||||||
|
//
|
||||||
|
// Mutation that must make this test fail: drop either mirror key from
|
||||||
|
// modelObject, or let one carry a different value from context_window.
|
||||||
|
func TestModelObject_ContextDialectMirrors(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
known := modelObject("m", "prov", resolvedModelMeta{ContextTokens: 1048576, ContextKnown: true})
|
||||||
|
for _, key := range []string{"context_window", "context_length", "max_model_len"} {
|
||||||
|
if known[key] != 1048576 {
|
||||||
|
t.Errorf("known[%q] = %v, want 1048576", key, known[key])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unknown := modelObject("m", "prov", resolvedModelMeta{})
|
||||||
|
for _, key := range []string{"context_window", "context_length", "max_model_len"} {
|
||||||
|
if v, present := unknown[key]; present {
|
||||||
|
t.Errorf("unknown[%q] present (%v), want omitted entirely", key, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user