diff --git a/provider_openai.go b/provider_openai.go index b3f102e..a11928e 100644 --- a/provider_openai.go +++ b/provider_openai.go @@ -426,20 +426,30 @@ func (a *openaiAdapter) listModels(ctx context.Context) ([]string, error) { return ids, nil } -// lmStudioModelsPayload is the wire shape of LM Studio's own, native -// models endpoint (ProviderConfig.MetadataPath — a real deployment sets -// it to "/api/v0/models"), verified live against a running LM Studio -// instance (2026-08): {"data":[{"id":...,"max_context_length":262144, -// "loaded_context_length":4096}, ...]}. Distinct from modelsPayload -// above, which is OpenAI's own /v1/models shape (LM Studio also serves -// that, unchanged, but it carries no per-model context metadata) — this -// is LM Studio's separate, additional endpoint this feature (v0.23) -// opts into reading. -type lmStudioModelsPayload struct { +// modelMetadataPayload is the wire shape read from +// ProviderConfig.MetadataPath. It carries two upstream dialects, because +// the runtimes that report per-model context do not agree on a field +// name: +// +// - LM Studio's own native endpoint (a real deployment sets +// MetadataPath to "/api/v0/models"), verified live against a running +// instance (2026-08): {"data":[{"id":...,"max_context_length":262144, +// "loaded_context_length":4096}, ...]}. +// - vLLM's OpenAI-compatible /v1/models, verified live against vLLM +// 0.28 (2026-09): {"data":[{"id":...,"max_model_len":8192, ...}]}. +// vLLM reports neither LM Studio field, so before max_model_len was +// read here a vLLM-backed provider had to have every model's context +// pinned by hand in modelMeta, and any model rename or window change +// silently served a stale number. +// +// Distinct from modelsPayload above, which is the same /v1/models +// endpoint decoded for ids alone. +type modelMetadataPayload struct { Data []struct { ID string `json:"id"` MaxContextLength int `json:"max_context_length"` LoadedContextLength int `json:"loaded_context_length"` + MaxModelLen int `json:"max_model_len"` } `json:"data"` } @@ -449,7 +459,8 @@ type lmStudioModelsPayload struct { // It prefers loaded_context_length when the upstream reports one greater // than zero — the context actually usable right now, which a runtime can // configure smaller than the model's own maximum under VRAM-constrained -// settings — and falls back to max_context_length otherwise. a. +// settings — and otherwise takes the widest declared window, which is +// max_context_length on LM Studio and max_model_len on vLLM. a. // metadataPath left empty (the default; every provider except an // operator's opt-in) is a fast no-op: (nil, nil), never an error, so // registry.go's captureModelMetadata can invoke this unconditionally on @@ -477,14 +488,23 @@ func (a *openaiAdapter) fetchModelMetadata(ctx context.Context) (map[string]int, return nil, fmt.Errorf("%w: read metadata response body: %w", errUpstream, err) } - var payload lmStudioModelsPayload + var payload modelMetadataPayload if err := json.Unmarshal(body, &payload); err != nil { return nil, fmt.Errorf("%w: decode metadata response: %w", errUpstream, err) } out := make(map[string]int, len(payload.Data)) for _, d := range payload.Data { + // Widest declared window first, by dialect: LM Studio's + // max_context_length, else vLLM's max_model_len. An upstream + // reports one or the other, never both. ctxLen := d.MaxContextLength + if ctxLen == 0 { + ctxLen = d.MaxModelLen + } + // loaded_context_length still wins where it is reported: it is the + // context actually usable right now, which a runtime can configure + // smaller than the model's own maximum. if d.LoadedContextLength > 0 { ctxLen = d.LoadedContextLength } diff --git a/provider_openai_test.go b/provider_openai_test.go index 58d078d..894fea8 100644 --- a/provider_openai_test.go +++ b/provider_openai_test.go @@ -592,6 +592,64 @@ func TestOpenAIAdapter_FetchModelMetadata(t *testing.T) { } }) + t.Run("vLLM shaped fixture: reads max_model_len", func(t *testing.T) { + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + // Verified against vLLM 0.28 (2026-09): it reports max_model_len and + // neither LM Studio field, so before this dialect was read a + // vLLM-backed provider needed every context pinned by hand. + _, _ = w.Write([]byte(`{"data":[ + {"id":"deepseek-v4-flash-vision-exp","object":"model","max_model_len":1048576}, + {"id":"text-embedding-qwen3-0.6b","object":"model","max_model_len":8192}, + {"id":"no-context-at-all","object":"model"} + ]}`)) + })) + defer srv.Close() + + a := newOpenAIAdapter("gx10", srv.URL, "") + a.metadataPath = "/v1/models" + got, err := a.fetchModelMetadata(context.Background()) + if err != nil { + t.Fatalf("fetchModelMetadata: %v", err) + } + if gotPath != "/v1/models" { + t.Errorf("request path = %q, want /v1/models", gotPath) + } + + want := map[string]int{ + "deepseek-v4-flash-vision-exp": 1048576, + "text-embedding-qwen3-0.6b": 8192, + // "no-context-at-all" omitted: it declares no window in either dialect. + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %#v, want %#v", got, want) + } + }) + + t.Run("LM Studio fields win over max_model_len when an upstream reports both", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[ + {"id":"both-max","max_context_length":262144,"max_model_len":999}, + {"id":"loaded-still-wins","max_context_length":262144,"loaded_context_length":4096,"max_model_len":999} + ]}`)) + })) + defer srv.Close() + + a := newOpenAIAdapter("mixed", srv.URL, "") + a.metadataPath = "/api/v0/models" + got, err := a.fetchModelMetadata(context.Background()) + if err != nil { + t.Fatalf("fetchModelMetadata: %v", err) + } + want := map[string]int{"both-max": 262144, "loaded-still-wins": 4096} + if !reflect.DeepEqual(got, want) { + t.Errorf("got %#v, want %#v", got, want) + } + }) + t.Run("non-2xx response is an error, not a captured empty map", func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError)