feat(metadata): read vLLM's max_model_len as a context dialect

The metadata fetcher understood only LM Studio's field names, so a
vLLM-backed provider reported no per-model context and every model had to
have its window pinned by hand in modelMeta. That pin then goes stale
silently: a renamed served model ships no context at all and clients fall
back to their own default, and a changed window advertises a size the
engine no longer accepts. Both happened on a live deployment today.

vLLM reports max_model_len at its OpenAI-compatible /v1/models and neither
LM Studio field, verified against vLLM 0.28. Reading it means pointing
MetadataPath at /v1/models is enough for such a provider.

Precedence is unchanged where it already applied: loaded_context_length
still wins as the window usable right now, then max_context_length, and
max_model_len is consulted only when neither is present. An upstream
reports one dialect or the other, so the added branch cannot change any
existing capture.

Renames the payload type, which now carries two dialects rather than LM
Studio's alone.
This commit is contained in:
2026-09-03 23:31:11 +01:00
parent 0f6ac09ce9
commit f530a1be61
2 changed files with 90 additions and 12 deletions
+58
View File
@@ -592,6 +592,64 @@ func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) {
}
})
t.Run("vLLM shaped fixture: reads max_model_len", func(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
// Verified against vLLM 0.28 (2026-09): it reports max_model_len and
// neither LM Studio field, so before this dialect was read a
// vLLM-backed provider needed every context pinned by hand.
_, _ = w.Write([]byte(`{"data":[
{"id":"deepseek-v4-flash-vision-exp","object":"model","max_model_len":1048576},
{"id":"text-embedding-qwen3-0.6b","object":"model","max_model_len":8192},
{"id":"no-context-at-all","object":"model"}
]}`))
}))
defer srv.Close()
a := newOpenAIAdapter("gx10", srv.URL, "")
a.metadataPath = "/v1/models"
got, err := a.fetchModelMetadata(context.Background())
if err != nil {
t.Fatalf("fetchModelMetadata: %v", err)
}
if gotPath != "/v1/models" {
t.Errorf("request path = %q, want /v1/models", gotPath)
}
want := map[string]int{
"deepseek-v4-flash-vision-exp": 1048576,
"text-embedding-qwen3-0.6b": 8192,
// "no-context-at-all" omitted: it declares no window in either dialect.
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %#v, want %#v", got, want)
}
})
t.Run("LM Studio fields win over max_model_len when an upstream reports both", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[
{"id":"both-max","max_context_length":262144,"max_model_len":999},
{"id":"loaded-still-wins","max_context_length":262144,"loaded_context_length":4096,"max_model_len":999}
]}`))
}))
defer srv.Close()
a := newOpenAIAdapter("mixed", srv.URL, "")
a.metadataPath = "/api/v0/models"
got, err := a.fetchModelMetadata(context.Background())
if err != nil {
t.Fatalf("fetchModelMetadata: %v", err)
}
want := map[string]int{"both-max": 262144, "loaded-still-wins": 4096}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %#v, want %#v", got, want)
}
})
t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)