mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-22 06:06:59 +00:00
feat(metadata): read vLLM's max_model_len as a context dialect
The metadata fetcher understood only LM Studio's field names, so a vLLM-backed provider reported no per-model context and every model had to have its window pinned by hand in modelMeta. That pin then goes stale silently: a renamed served model ships no context at all and clients fall back to their own default, and a changed window advertises a size the engine no longer accepts. Both happened on a live deployment today. vLLM reports max_model_len at its OpenAI-compatible /v1/models and neither LM Studio field, verified against vLLM 0.28. Reading it means pointing MetadataPath at /v1/models is enough for such a provider. Precedence is unchanged where it already applied: loaded_context_length still wins as the window usable right now, then max_context_length, and max_model_len is consulted only when neither is present. An upstream reports one dialect or the other, so the added branch cannot change any existing capture. Renames the payload type, which now carries two dialects rather than LM Studio's alone.
This commit is contained in:
@@ -592,6 +592,64 @@ func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("vLLM shaped fixture: reads max_model_len", func(t *testing.T) {
|
||||
var gotPath string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
// Verified against vLLM 0.28 (2026-09): it reports max_model_len and
|
||||
// neither LM Studio field, so before this dialect was read a
|
||||
// vLLM-backed provider needed every context pinned by hand.
|
||||
_, _ = w.Write([]byte(`{"data":[
|
||||
{"id":"deepseek-v4-flash-vision-exp","object":"model","max_model_len":1048576},
|
||||
{"id":"text-embedding-qwen3-0.6b","object":"model","max_model_len":8192},
|
||||
{"id":"no-context-at-all","object":"model"}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("gx10", srv.URL, "")
|
||||
a.metadataPath = "/v1/models"
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetchModelMetadata: %v", err)
|
||||
}
|
||||
if gotPath != "/v1/models" {
|
||||
t.Errorf("request path = %q, want /v1/models", gotPath)
|
||||
}
|
||||
|
||||
want := map[string]int{
|
||||
"deepseek-v4-flash-vision-exp": 1048576,
|
||||
"text-embedding-qwen3-0.6b": 8192,
|
||||
// "no-context-at-all" omitted: it declares no window in either dialect.
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("LM Studio fields win over max_model_len when an upstream reports both", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"data":[
|
||||
{"id":"both-max","max_context_length":262144,"max_model_len":999},
|
||||
{"id":"loaded-still-wins","max_context_length":262144,"loaded_context_length":4096,"max_model_len":999}
|
||||
]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
a := newOpenAIAdapter("mixed", srv.URL, "")
|
||||
a.metadataPath = "/api/v0/models"
|
||||
got, err := a.fetchModelMetadata(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("fetchModelMetadata: %v", err)
|
||||
}
|
||||
want := map[string]int{"both-max": 262144, "loaded-still-wins": 4096}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("got %#v, want %#v", got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
|
||||
Reference in New Issue
Block a user