feat(metadata): read vLLM's max_model_len as a context dialect

The metadata fetcher understood only LM Studio's field names, so a
vLLM-backed provider reported no per-model context and every model had to
have its window pinned by hand in modelMeta. That pin then goes stale
silently: a renamed served model ships no context at all and clients fall
back to their own default, and a changed window advertises a size the
engine no longer accepts. Both happened on a live deployment today.

vLLM reports max_model_len at its OpenAI-compatible /v1/models and neither
LM Studio field, verified against vLLM 0.28. Reading it means pointing
MetadataPath at /v1/models is enough for such a provider.

Precedence is unchanged where it already applied: loaded_context_length
still wins as the window usable right now, then max_context_length, and
max_model_len is consulted only when neither is present. An upstream
reports one dialect or the other, so the added branch cannot change any
existing capture.

Renames the payload type, which now carries two dialects rather than LM
Studio's alone.
This commit is contained in:
2026-09-03 23:31:11 +01:00
parent 0f6ac09ce9
commit f530a1be61
2 changed files with 90 additions and 12 deletions
+32 -12
View File
@@ -426,20 +426,30 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) {
return ids, nil
}
// lmStudioModelsPayload is the wire shape of LM Studio's own, native
// models endpoint (ProviderConfig.MetadataPath — a real deployment sets
// it to "/api/v0/models"), verified live against a running LM Studio
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload
// above, which is OpenAI's own /v1/models shape (LM Studio also serves
// that, unchanged, but it carries no per-model context metadata) — this
// is LM Studio's separate, additional endpoint this feature (v0.23)
// opts into reading.
type lmStudioModelsPayload struct {
// modelMetadataPayload is the wire shape read from
// ProviderConfig.MetadataPath. It carries two upstream dialects, because
// the runtimes that report per-model context do not agree on a field
// name:
//
// - LM Studio's own native endpoint (a real deployment sets
// MetadataPath to "/api/v0/models"), verified live against a running
// instance (2026-08): {"data":[{"id":...,"max_context_length":262144,
// "loaded_context_length":4096}, ...]}.
// - vLLM's OpenAI-compatible /v1/models, verified live against vLLM
// 0.28 (2026-09): {"data":[{"id":...,"max_model_len":8192, ...}]}.
// vLLM reports neither LM Studio field, so before max_model_len was
// read here a vLLM-backed provider had to have every model's context
// pinned by hand in modelMeta, and any model rename or window change
// silently served a stale number.
//
// Distinct from modelsPayload above, which is the same /v1/models
// endpoint decoded for ids alone.
type modelMetadataPayload struct {
Data []struct {
ID string `json:"id"`
MaxContextLength int `json:"max_context_length"`
LoadedContextLength int `json:"loaded_context_length"`
MaxModelLen int `json:"max_model_len"`
} `json:"data"`
}
@@ -449,7 +459,8 @@ type lmStudioModelsPayload struct {
// It prefers loaded_context_length when the upstream reports one greater
// than zero — the context actually usable right now, which a runtime can
// configure smaller than the model's own maximum under VRAM-constrained
// settings — and falls back to max_context_length otherwise. a.
// settings — and otherwise takes the widest declared window, which is
// max_context_length on LM Studio and max_model_len on vLLM. a.
// metadataPath left empty (the default; every provider except an
// operator's opt-in) is a fast no-op: (nil, nil), never an error, so
// registry.go's captureModelMetadata can invoke this unconditionally on
@@ -477,14 +488,23 @@ func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int,
return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err)
}
var payload lmStudioModelsPayload
var payload modelMetadataPayload
if err := json.Unmarshal(body, &payload); err != nil {
return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err)
}
out := make(map[string]int, len(payload.Data))
for _, d := range payload.Data {
// Widest declared window first, by dialect: LM Studio's
// max_context_length, else vLLM's max_model_len. An upstream
// reports one or the other, never both.
ctxLen := d.MaxContextLength
if ctxLen == 0 {
ctxLen = d.MaxModelLen
}
// loaded_context_length still wins where it is reported: it is the
// context actually usable right now, which a runtime can configure
// smaller than the model's own maximum.
if d.LoadedContextLength > 0 {
ctxLen = d.LoadedContextLength
}
+58
View File
@@ -592,6 +592,64 @@ func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) {
}
})
t.Run("vLLM shaped fixture: reads max_model_len", func(t *testing.T) {
var gotPath string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
// Verified against vLLM 0.28 (2026-09): it reports max_model_len and
// neither LM Studio field, so before this dialect was read a
// vLLM-backed provider needed every context pinned by hand.
_, _ = w.Write([]byte(`{"data":[
{"id":"deepseek-v4-flash-vision-exp","object":"model","max_model_len":1048576},
{"id":"text-embedding-qwen3-0.6b","object":"model","max_model_len":8192},
{"id":"no-context-at-all","object":"model"}
]}`))
}))
defer srv.Close()
a := newOpenAIAdapter("gx10", srv.URL, "")
a.metadataPath = "/v1/models"
got, err := a.fetchModelMetadata(context.Background())
if err != nil {
t.Fatalf("fetchModelMetadata: %v", err)
}
if gotPath != "/v1/models" {
t.Errorf("request path = %q, want /v1/models", gotPath)
}
want := map[string]int{
"deepseek-v4-flash-vision-exp": 1048576,
"text-embedding-qwen3-0.6b": 8192,
// "no-context-at-all" omitted: it declares no window in either dialect.
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %#v, want %#v", got, want)
}
})
t.Run("LM Studio fields win over max_model_len when an upstream reports both", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":[
{"id":"both-max","max_context_length":262144,"max_model_len":999},
{"id":"loaded-still-wins","max_context_length":262144,"loaded_context_length":4096,"max_model_len":999}
]}`))
}))
defer srv.Close()
a := newOpenAIAdapter("mixed", srv.URL, "")
a.metadataPath = "/api/v0/models"
got, err := a.fetchModelMetadata(context.Background())
if err != nil {
t.Fatalf("fetchModelMetadata: %v", err)
}
want := map[string]int{"both-max": 262144, "loaded-still-wins": 4096}
if !reflect.DeepEqual(got, want) {
t.Errorf("got %#v, want %#v", got, want)
}
})
t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)