mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-17 05:37:43 +00:00
feat(metadata): read vLLM's max_model_len as a context dialect
The metadata fetcher understood only LM Studio's field names, so a vLLM-backed provider reported no per-model context and every model had to have its window pinned by hand in modelMeta. That pin then goes stale silently: a renamed served model ships no context at all and clients fall back to their own default, and a changed window advertises a size the engine no longer accepts. Both happened on a live deployment today. vLLM reports max_model_len at its OpenAI-compatible /v1/models and neither LM Studio field, verified against vLLM 0.28. Reading it means pointing MetadataPath at /v1/models is enough for such a provider. Precedence is unchanged where it already applied: loaded_context_length still wins as the window usable right now, then max_context_length, and max_model_len is consulted only when neither is present. An upstream reports one dialect or the other, so the added branch cannot change any existing capture. Renames the payload type, which now carries two dialects rather than LM Studio's alone.
This commit is contained in:
+32
-12
@@ -426,20 +426,30 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
// lmStudioModelsPayload is the wire shape of LM Studio's own, native
|
||||
// models endpoint (ProviderConfig.MetadataPath — a real deployment sets
|
||||
// it to "/api/v0/models"), verified live against a running LM Studio
|
||||
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
|
||||
// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload
|
||||
// above, which is OpenAI's own /v1/models shape (LM Studio also serves
|
||||
// that, unchanged, but it carries no per-model context metadata) — this
|
||||
// is LM Studio's separate, additional endpoint this feature (v0.23)
|
||||
// opts into reading.
|
||||
type lmStudioModelsPayload struct {
|
||||
// modelMetadataPayload is the wire shape read from
|
||||
// ProviderConfig.MetadataPath. It carries two upstream dialects, because
|
||||
// the runtimes that report per-model context do not agree on a field
|
||||
// name:
|
||||
//
|
||||
// - LM Studio's own native endpoint (a real deployment sets
|
||||
// MetadataPath to "/api/v0/models"), verified live against a running
|
||||
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
|
||||
// "loaded_context_length":4096}, ...]}.
|
||||
// - vLLM's OpenAI-compatible /v1/models, verified live against vLLM
|
||||
// 0.28 (2026-09): {"data":[{"id":...,"max_model_len":8192, ...}]}.
|
||||
// vLLM reports neither LM Studio field, so before max_model_len was
|
||||
// read here a vLLM-backed provider had to have every model's context
|
||||
// pinned by hand in modelMeta, and any model rename or window change
|
||||
// silently served a stale number.
|
||||
//
|
||||
// Distinct from modelsPayload above, which is the same /v1/models
|
||||
// endpoint decoded for ids alone.
|
||||
type modelMetadataPayload struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
MaxContextLength int `json:"max_context_length"`
|
||||
LoadedContextLength int `json:"loaded_context_length"`
|
||||
MaxModelLen int `json:"max_model_len"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
@@ -449,7 +459,8 @@ type lmStudioModelsPayload struct {
|
||||
// It prefers loaded_context_length when the upstream reports one greater
|
||||
// than zero — the context actually usable right now, which a runtime can
|
||||
// configure smaller than the model's own maximum under VRAM-constrained
|
||||
// settings — and falls back to max_context_length otherwise. a.
|
||||
// settings — and otherwise takes the widest declared window, which is
|
||||
// max_context_length on LM Studio and max_model_len on vLLM. a.
|
||||
// metadataPath left empty (the default; every provider except an
|
||||
// operator's opt-in) is a fast no-op: (nil, nil), never an error, so
|
||||
// registry.go's captureModelMetadata can invoke this unconditionally on
|
||||
@@ -477,14 +488,23 @@ func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int,
|
||||
return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
var payload lmStudioModelsPayload
|
||||
var payload modelMetadataPayload
|
||||
if err := json.Unmarshal(body, &payload); err != nil {
|
||||
return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err)
|
||||
}
|
||||
|
||||
out := make(map[string]int, len(payload.Data))
|
||||
for _, d := range payload.Data {
|
||||
// Widest declared window first, by dialect: LM Studio's
|
||||
// max_context_length, else vLLM's max_model_len. An upstream
|
||||
// reports one or the other, never both.
|
||||
ctxLen := d.MaxContextLength
|
||||
if ctxLen == 0 {
|
||||
ctxLen = d.MaxModelLen
|
||||
}
|
||||
// loaded_context_length still wins where it is reported: it is the
|
||||
// context actually usable right now, which a runtime can configure
|
||||
// smaller than the model's own maximum.
|
||||
if d.LoadedContextLength > 0 {
|
||||
ctxLen = d.LoadedContextLength
|
||||
}
|
||||
|
||||
@@ -592,6 +592,64 @@ func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("vLLM shaped fixture: reads max_model_len", func(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
// Verified against vLLM 0.28 (2026-09): it reports max_model_len and
|
||||
// neither LM Studio field, so before this dialect was read a
|
||||
// vLLM-backed provider needed every context pinned by hand.
|
||||
_, _ = w.Write([]byte(`{"data":[
|
||||
{"id":"deepseek-v4-flash-vision-exp","object":"model","max_model_len":1048576},
|
||||
{"id":"text-embedding-qwen3-0.6b","object":"model","max_model_len":8192},
|
||||
{"id":"no-context-at-all","object":"model"}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("gx10", srv.URL, "")
|
||||
a.metadataPath = "/v1/models"
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetchModelMetadata: %v", err)
|
||||
}
|
||||
if gotPath != "/v1/models" {
|
||||
t.Errorf("request path = %q, want /v1/models", gotPath)
|
||||
}
|
||||
|
||||
want := map[string]int{
|
||||
"deepseek-v4-flash-vision-exp": 1048576,
|
||||
"text-embedding-qwen3-0.6b": 8192,
|
||||
// "no-context-at-all" omitted: it declares no window in either dialect.
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("LM Studio fields win over max_model_len when an upstream reports both", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[
|
||||
{"id":"both-max","max_context_length":262144,"max_model_len":999},
|
||||
{"id":"loaded-still-wins","max_context_length":262144,"loaded_context_length":4096,"max_model_len":999}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("mixed", srv.URL, "")
|
||||
a.metadataPath = "/api/v0/models"
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetchModelMetadata: %v", err)
|
||||
}
|
||||
want := map[string]int{"both-max": 262144, "loaded-still-wins": 4096}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
|
||||
Reference in New Issue
Block a user