mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-12 04:54:45 +00:00
fix(provider): list models in one page for Anthropic and Gemini
Model discovery used each API's default page size: 20 for Anthropic and 50 for Gemini. A full catalog needed several sequential requests, and at pod start that overran the 5s warm-fill budget. One replica then logged "initial discovery ... context deadline exceeded" and went without that provider's models until the next probe. Every page now asks for the documented maximum of 1000: Anthropic limit and Gemini pageSize. Gemini gets pageSize on each page, because it requires the same parameters across one listing. Claude-Session: https://claude.ai/code/session_01V5jYJNxpDFdJkyyExpADrv
This commit is contained in:
+15
-5
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -270,11 +271,18 @@ type anthropicModelsPayload struct {
|
||||
// anthropicListModelsPageCap bounds how many pages listModels follows
|
||||
// (M7 fix): a misbehaving upstream that never reports has_more:false, or
|
||||
// whose last_id never advances, would otherwise page forever. 20 pages
|
||||
// at Anthropic's own default page size (20 per page) covers 400 models —
|
||||
// comfortably above any real catalog as of this writing — without
|
||||
// risking an unbounded discovery loop against a single provider.
|
||||
// at anthropicListModelsPageSize covers far more models than any real
|
||||
// catalog without risking an unbounded discovery loop against a single
|
||||
// provider.
|
||||
const anthropicListModelsPageCap = 20
|
||||
|
||||
// anthropicListModelsPageSize is the `limit` sent on every /v1/models
|
||||
// page: Anthropic's documented maximum (default 20, range 1-1000). At the
|
||||
// default page size a full catalog took several sequential requests, which
|
||||
// could overrun registry.go's warmFillTimeout at pod start; one request
|
||||
// at the maximum returns the whole catalog.
|
||||
const anthropicListModelsPageSize = 1000
|
||||
|
||||
// listModels implements providerAdapter: GET {base}/v1/models, following
|
||||
// Anthropic's has_more/last_id cursor pagination (M7 fix) via the
|
||||
// after_id query parameter, up to anthropicListModelsPageCap pages —
|
||||
@@ -307,10 +315,12 @@ func (a *anthropicAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
// model ids alongside has_more/last_id for listModels' own pagination
|
||||
// loop above.
|
||||
func (a *anthropicAdapter) listModelsPage(ctx context.Context, afterID string) (ids []string, hasMore bool, lastID string, err error) {
|
||||
endpoint := a.baseURL + anthropicModelsPath
|
||||
query := url.Values{}
|
||||
query.Set("limit", strconv.Itoa(anthropicListModelsPageSize))
|
||||
if afterID != "" {
|
||||
endpoint += "?after_id=" + url.QueryEscape(afterID)
|
||||
query.Set("after_id", afterID)
|
||||
}
|
||||
endpoint := a.baseURL + anthropicModelsPath + "?" + query.Encode()
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, endpoint, a.requestHeaders(false), nil, a.retry, a.timeout, a.adapterName)
|
||||
if err != nil {
|
||||
return nil, false, "", err
|
||||
|
||||
@@ -471,9 +471,10 @@ func TestAnthropicAdapter_ListModels(t *testing.T) {
|
||||
// page via ?after_id=<last_id> and return the union of both pages, not
|
||||
// just the first page's 20 (Anthropic's own default page size).
|
||||
func TestAnthropicAdapter_ListModels_FollowsPagination(t *testing.T) {
|
||||
var gotAfterIDs []string
|
||||
var gotAfterIDs, gotLimits []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAfterIDs = append(gotAfterIDs, r.URL.Query().Get("after_id"))
|
||||
gotLimits = append(gotLimits, r.URL.Query().Get("limit"))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Query().Get("after_id") == "" {
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"claude-opus-5"},{"id":"claude-sonnet-5"}],"has_more":true,"last_id":"claude-sonnet-5"}`))
|
||||
@@ -504,6 +505,13 @@ func TestAnthropicAdapter_ListModels_FollowsPagination(t *testing.T) {
|
||||
if len(gotAfterIDs) != 2 || gotAfterIDs[0] != "" || gotAfterIDs[1] != "claude-sonnet-5" {
|
||||
t.Errorf("after_id sequence = %#v, want [\"\", \"claude-sonnet-5\"] (first page unqualified, second page's after_id = first page's last_id)", gotAfterIDs)
|
||||
}
|
||||
// Every page asks for Anthropic's maximum page size, so a real catalog
|
||||
// arrives in one request inside warmFillTimeout.
|
||||
for i, l := range gotLimits {
|
||||
if l != "1000" {
|
||||
t.Errorf("page %d limit = %q, want \"1000\"", i, l)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnthropicAdapter_ListModels_PageCapStopsRunawayPagination proves
|
||||
|
||||
+12
-2
@@ -7,6 +7,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -386,6 +387,13 @@ type geminiModelsListPayload struct {
|
||||
// provider.
|
||||
const geminiListModelsPageCap = 20
|
||||
|
||||
// geminiListModelsPageSize is the pageSize sent on every models.list page:
|
||||
// the documented maximum (default 50, at most 1000 per page). Gemini
|
||||
// requires the same parameters on every page of one listing, so it is sent
|
||||
// with the pageToken too. One request at the maximum returns the whole
|
||||
// catalog within registry.go's warmFillTimeout.
|
||||
const geminiListModelsPageSize = 1000
|
||||
|
||||
// listModels implements providerAdapter: GET {base}/v1beta/models,
|
||||
// stripping the "models/" prefix off each entry's "name" to return bare
|
||||
// model ids, matching the shape openai-type and anthropic-type adapters
|
||||
@@ -414,10 +422,12 @@ func (a *geminiAdapter) listModels(ctx context.Context) ([]string, error) {
|
||||
// model ids alongside nextPageToken for listModels' own pagination loop
|
||||
// above.
|
||||
func (a *geminiAdapter) listModelsPage(ctx context.Context, pageToken string) (ids []string, nextPageToken string, err error) {
|
||||
endpoint := a.baseURL + "/v1beta/models"
|
||||
query := url.Values{}
|
||||
query.Set("pageSize", strconv.Itoa(geminiListModelsPageSize))
|
||||
if pageToken != "" {
|
||||
endpoint += "?pageToken=" + url.QueryEscape(pageToken)
|
||||
query.Set("pageToken", pageToken)
|
||||
}
|
||||
endpoint := a.baseURL + "/v1beta/models?" + query.Encode()
|
||||
resp, err := upstreamJSON(ctx, a.client, http.MethodGet, endpoint, a.requestHeaders(false), nil, a.retry, a.timeout, a.adapterName)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
|
||||
@@ -688,9 +688,10 @@ func TestGeminiAdapter_ListModels(t *testing.T) {
|
||||
// test: a non-empty nextPageToken must make listModels fetch a second
|
||||
// page via ?pageToken=<token> and return the union of both pages.
|
||||
func TestGeminiAdapter_ListModels_FollowsPagination(t *testing.T) {
|
||||
var gotPageTokens []string
|
||||
var gotPageTokens, gotPageSizes []string
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPageTokens = append(gotPageTokens, r.URL.Query().Get("pageToken"))
|
||||
gotPageSizes = append(gotPageSizes, r.URL.Query().Get("pageSize"))
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if r.URL.Query().Get("pageToken") == "" {
|
||||
_, _ = w.Write([]byte(`{"models":[{"name":"models/gemini-2.5-pro"},{"name":"models/gemini-2.5-flash"}],"nextPageToken":"page2"}`))
|
||||
@@ -721,6 +722,13 @@ func TestGeminiAdapter_ListModels_FollowsPagination(t *testing.T) {
|
||||
if len(gotPageTokens) != 2 || gotPageTokens[0] != "" || gotPageTokens[1] != "page2" {
|
||||
t.Errorf("pageToken sequence = %#v, want [\"\", \"page2\"] (first page unqualified, second page's pageToken = first page's nextPageToken)", gotPageTokens)
|
||||
}
|
||||
// Gemini requires identical parameters on every page of one listing,
|
||||
// so pageSize must ride along with the pageToken too.
|
||||
for i, s := range gotPageSizes {
|
||||
if s != "1000" {
|
||||
t.Errorf("page %d pageSize = %q, want \"1000\"", i, s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGeminiAdapter_ListModels_PageCapStopsRunawayPagination proves the
|
||||
|
||||
Reference in New Issue
Block a user