mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-18 05:47:38 +00:00
test(yaegi-check): exercise the builtin table lookup (review SHOULD-6)
The harness previously only ever proved resolveModelMeta's CONFIG- OVERRIDE layer runs interpreted (testDataWantModel carries an explicit modelMeta entry). It never exercised an interpreted LOOKUP into the real, 236-entry builtinModelMetaTable map literal at all. Adds "gpt-4o" to the harness's openai provider with NO modelMeta override of its own, so its GET /v1/models context_window can only come from the builtin-table branch. Its expected value is read directly out of pricing_data_gen.go at harness run time (readBuiltinContextTokens), not hardcoded, so this assertion self-updates across a `make pricing-sync` regeneration instead of pinning a number that could legitimately drift when upstream LiteLLM data changes. Also switched both existing GET /v1/models assertions from raw strings.Contains substring matching to a structured JSON decode keyed by model id — a bare substring check could not tell which model entry a given context_window value actually belonged to, which now matters with two entries in play.
This commit is contained in:
+106
-15
@@ -34,6 +34,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -225,7 +226,17 @@ func run() error {
|
|||||||
}))
|
}))
|
||||||
defer slowUpstream.Close()
|
defer slowUpstream.Close()
|
||||||
|
|
||||||
attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `"]},"` +
|
// builtinLookupContextTokens (review fix, SHOULD-6): read directly
|
||||||
|
// out of the generated pricing_data_gen.go rather than hardcoding a
|
||||||
|
// number, so this assertion self-updates across a `make
|
||||||
|
// pricing-sync` regeneration instead of pinning a value that could
|
||||||
|
// legitimately drift when LiteLLM's own upstream data changes.
|
||||||
|
builtinLookupContextTokens, err := readBuiltinContextTokens(repoRoot, builtinLookupModelID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("read builtin context tokens for %q from pricing_data_gen.go: %w", builtinLookupModelID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
attemptAccountingOverride := `{"providers":{"openai":{"type":"openai","baseUrl":"` + upstream.URL + `","apiKey":"sk-up","models":["` + testDataWantModel + `","` + builtinLookupModelID + `"]},"` +
|
||||||
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
|
slowProviderName + `":{"type":"openai","baseUrl":"` + slowUpstream.URL + `","apiKey":"sk-up","models":["` + slowProviderModel + `"]}},` +
|
||||||
// modelMeta (feature v0.23): a config-override entry for
|
// modelMeta (feature v0.23): a config-override entry for
|
||||||
// testDataWantModel, so exerciseHandler's GET /v1/models
|
// testDataWantModel, so exerciseHandler's GET /v1/models
|
||||||
@@ -234,7 +245,12 @@ func run() error {
|
|||||||
// and the whole registry.go/modelmeta.go resolution path all
|
// and the whole registry.go/modelmeta.go resolution path all
|
||||||
// run correctly INTERPRETED, not merely compiled — cheap to add
|
// run correctly INTERPRETED, not merely compiled — cheap to add
|
||||||
// to this existing harness request, per this feature's own gate
|
// to this existing harness request, per this feature's own gate
|
||||||
// requirement.
|
// requirement. builtinLookupModelID carries NO config override
|
||||||
|
// at all (SHOULD-6, review round) — its context_window can only
|
||||||
|
// come from an interpreted LOOKUP into the real, 200+-entry
|
||||||
|
// builtinModelMetaTable map literal, proving that specific path
|
||||||
|
// runs correctly under Yaegi too, not just the config-override
|
||||||
|
// one testDataWantModel already exercises.
|
||||||
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
|
`"modelMeta":{"` + testDataWantModel + `":{"contextTokens":` + yaegiMetaContextTokens + `,"inputCostPerMTokMicroUsd":1250000,"outputCostPerMTokMicroUsd":10000000}},` +
|
||||||
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
|
`"admin":{"enabled":true},"users":{"inline":[{"name":"tester","group":"default","apiKey":"` + testDataUserAPIKey + `"},{"name":"admin1","group":"default","apiKey":"` + attemptAccountingAdminAPIKey + `","admin":true}]}}`
|
||||||
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
|
if err = json.Unmarshal([]byte(attemptAccountingOverride), cfgVal.Interface()); err != nil {
|
||||||
@@ -271,7 +287,41 @@ func run() error {
|
|||||||
return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName)
|
return fmt.Errorf("%s.New's returned value does not implement http.Handler", pkgName)
|
||||||
}
|
}
|
||||||
|
|
||||||
return exerciseHandler(handler)
|
return exerciseHandler(handler, builtinLookupContextTokens)
|
||||||
|
}
|
||||||
|
|
||||||
|
// builtinLookupModelID is a real, stable entry in the generated
|
||||||
|
// builtinModelMetaTable (pricing_data_gen.go) — review fix, SHOULD-6 —
|
||||||
|
// added to the harness's openai provider with NO modelMeta config
|
||||||
|
// override of its own, so its GET /v1/models context_window can only
|
||||||
|
// ever come from an interpreted lookup into the real map literal, not
|
||||||
|
// the config-override path testDataWantModel already exercises.
|
||||||
|
const builtinLookupModelID = "gpt-4o"
|
||||||
|
|
||||||
|
// readBuiltinContextTokens reads repoRoot/pricing_data_gen.go and
|
||||||
|
// extracts modelID's own ContextTokens value directly out of the
|
||||||
|
// generated source (review fix, SHOULD-6) — rather than hardcoding a
|
||||||
|
// number in this harness, which would need hand-updating (and could
|
||||||
|
// silently drift out of sync) every time `make pricing-sync`
|
||||||
|
// regenerates the table against updated upstream data.
|
||||||
|
func readBuiltinContextTokens(repoRoot, modelID string) (int, error) {
|
||||||
|
data, err := os.ReadFile(filepath.Join(repoRoot, "pricing_data_gen.go"))
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("read pricing_data_gen.go: %w", err)
|
||||||
|
}
|
||||||
|
pattern := regexp.MustCompile(regexp.QuoteMeta(`"`+modelID+`":`) + `\s*\{ContextTokens:\s*(\d+)`)
|
||||||
|
m := pattern.FindSubmatch(data)
|
||||||
|
if m == nil {
|
||||||
|
return 0, fmt.Errorf("no builtinModelMetaTable entry found for %q — pick a different, currently-present stable id", modelID)
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(string(m[1]))
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("parse ContextTokens for %q: %w", modelID, err)
|
||||||
|
}
|
||||||
|
if n <= 0 {
|
||||||
|
return 0, fmt.Errorf("builtinModelMetaTable entry for %q has ContextTokens=%d, want > 0", modelID, n)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// exerciseHandler runs two real requests against the interpreted
|
// exerciseHandler runs two real requests against the interpreted
|
||||||
@@ -281,7 +331,10 @@ func run() error {
|
|||||||
// be refused with 401 — proving ServeHTTP, auth.identify, the model
|
// be refused with 401 — proving ServeHTTP, auth.identify, the model
|
||||||
// registry, statusTrackingWriter, and the gateway's logger all execute
|
// registry, statusTrackingWriter, and the gateway's logger all execute
|
||||||
// for real under Yaegi, not merely that the package's imports resolve.
|
// for real under Yaegi, not merely that the package's imports resolve.
|
||||||
func exerciseHandler(handler http.Handler) error {
|
// builtinContextTokens is builtinLookupModelID's own real, extracted
|
||||||
|
// builtinModelMetaTable value (SHOULD-6) — see that constant's own doc
|
||||||
|
// comment.
|
||||||
|
func exerciseHandler(handler http.Handler, builtinContextTokens int) error {
|
||||||
authedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
authedReq := httptest.NewRequest(http.MethodGet, "/v1/models", nil)
|
||||||
authedReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
authedReq.Header.Set("Authorization", "Bearer "+testDataUserAPIKey)
|
||||||
authedRec := httptest.NewRecorder()
|
authedRec := httptest.NewRecorder()
|
||||||
@@ -292,18 +345,56 @@ func exerciseHandler(handler http.Handler) error {
|
|||||||
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
|
if !strings.Contains(authedRec.Body.String(), testDataWantModel) {
|
||||||
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
|
return fmt.Errorf("GET /v1/models body does not contain %q: %s", testDataWantModel, authedRec.Body.String())
|
||||||
}
|
}
|
||||||
// modelMeta (feature v0.23), interpreted: run() layers a modelMeta
|
|
||||||
// config override for testDataWantModel onto cfgVal before New() is
|
// modelMeta (feature v0.23), interpreted, decoded structurally (not
|
||||||
// ever called (attemptAccountingOverride above) — proving under the
|
// a raw substring match — SHOULD-6, review round: a bare Contains
|
||||||
// real Yaegi interpreter, not just `go test`, that resolveModelMeta's
|
// check cannot tell WHICH model entry a given context_window value
|
||||||
// config-override layer and modelObject's context_window/pricing
|
// belongs to, which matters once two entries can legitimately share
|
||||||
// extension fields both run correctly through registry.go's
|
// the same number) so each assertion below is pinned to its own
|
||||||
// listFor/resolveMetaFor.
|
// named model id.
|
||||||
if !strings.Contains(authedRec.Body.String(), `"context_window":128000`) {
|
var modelsBody struct {
|
||||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved context_window (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
Data []map[string]any `json:"data"`
|
||||||
}
|
}
|
||||||
if !strings.Contains(authedRec.Body.String(), `"input_per_mtok_usd":1.25`) || !strings.Contains(authedRec.Body.String(), `"output_per_mtok_usd":10`) {
|
if err := json.Unmarshal(authedRec.Body.Bytes(), &modelsBody); err != nil {
|
||||||
return fmt.Errorf("GET /v1/models body does not contain the modelMeta-resolved pricing (feature v0.23 metadata harness): %s", authedRec.Body.String())
|
return fmt.Errorf("decode GET /v1/models body: %w (body=%s)", err, authedRec.Body.String())
|
||||||
|
}
|
||||||
|
byID := make(map[string]map[string]any, len(modelsBody.Data))
|
||||||
|
for _, entry := range modelsBody.Data {
|
||||||
|
if id, ok := entry["id"].(string); ok {
|
||||||
|
byID[id] = entry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// run() layers a modelMeta config override for testDataWantModel
|
||||||
|
// onto cfgVal before New() is ever called (attemptAccountingOverride
|
||||||
|
// above) — proving resolveModelMeta's config-override layer and
|
||||||
|
// modelObject's context_window/pricing extension fields both run
|
||||||
|
// correctly, interpreted, through registry.go's listFor/
|
||||||
|
// resolveMetaFor.
|
||||||
|
overridden, ok := byID[testDataWantModel]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", testDataWantModel, authedRec.Body.String())
|
||||||
|
}
|
||||||
|
if cw, _ := overridden["context_window"].(float64); cw != 128000 {
|
||||||
|
return fmt.Errorf("GET /v1/models %q context_window = %v, want 128000 (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["context_window"], authedRec.Body.String())
|
||||||
|
}
|
||||||
|
pricing, _ := overridden["pricing"].(map[string]any)
|
||||||
|
if pricing == nil || pricing["input_per_mtok_usd"] != 1.25 || pricing["output_per_mtok_usd"] != 10.0 {
|
||||||
|
return fmt.Errorf("GET /v1/models %q pricing = %v, want {input_per_mtok_usd:1.25 output_per_mtok_usd:10} (feature v0.23 config-override metadata harness): %s", testDataWantModel, overridden["pricing"], authedRec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// builtinLookupModelID (SHOULD-6, review round) carries NO modelMeta
|
||||||
|
// config override — its context_window can only come from an
|
||||||
|
// interpreted LOOKUP into the real, 200+-entry builtinModelMetaTable
|
||||||
|
// map literal (modelmeta.go's resolveModelMeta, builtin branch),
|
||||||
|
// proving that specific path — not just the config-override one
|
||||||
|
// above — runs correctly under Yaegi.
|
||||||
|
builtinLookedUp, ok := byID[builtinLookupModelID]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("GET /v1/models body has no entry for %q: %s", builtinLookupModelID, authedRec.Body.String())
|
||||||
|
}
|
||||||
|
if cw, _ := builtinLookedUp["context_window"].(float64); int(cw) != builtinContextTokens {
|
||||||
|
return fmt.Errorf("GET /v1/models %q context_window = %v, want %d (feature v0.23 builtin-table-lookup metadata harness, SHOULD-6): %s", builtinLookupModelID, builtinLookedUp["context_window"], builtinContextTokens, authedRec.Body.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := exerciseAttemptAccounting(handler); err != nil {
|
if err := exerciseAttemptAccounting(handler); err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user