mirror of
https://github.com/lukaszraczylo/traefik-llmgateway.git
synced 2026-10-21 05:57:53 +00:00
perf(limits): fuse admission reads into one round trip
checkAndCount already batched its request-counter increments into one pipeline. evaluateScope then read every scope's token and cost budgets through up to four further serial round trips per scope. A request with a fully-limited user and group scope paid nine round trips before the upstream call started. This change fuses every budget read into the SAME pipeline as the increments, through a new incrAndGetMulti method on counterStore. The reads are safe to fuse because they touch disjoint keys (req:* versus tokin:*/tokout:*/cost:*), and tokens and cost are only ever written by account() after the response completes, never by checkAndCount itself. Violation precedence stays identical to the pre-fusion implementation: a user's own violation still wins over their group's, and within one scope the order stays requests-per-minute, requests-per-day, tokens-per-day, tokens-per-month, cost-per-day, cost-per-month. New table-driven tests pin this precedence directly, engineering two simultaneous violation conditions per case and asserting only the higher-precedence one is reported. evaluateScope, budgetViolation, and tokenBudgetViolation are removed; buildBudgetProbes plus a loop inline the same logic into checkAndCount, which is what lets their reads share the fused round trip. Measured on the interpreted Yaegi harness (yaegi v0.16.1, the runtime this plugin actually ships under): RTT/req for a fully-limited chat request drops from 10.99 to 3.00, with cmds/req unchanged at 74 (this is a round-trip fix, not a command-count or allocation change). Single-request throughput rises from 112 to 335 req/s; at concurrency 16, from 121 to 380 req/s. counterStore.incrAndGetMulti is a required interface method, not an optional capability behind a type assertion: Yaegi can silently return ok=false when a comma-ok assertion checks a compiled concrete value against a plugin-declared interface (Trap 4, documented in matchesSentinel's own doc comment). A plain interface method call carries no such risk and is the same mechanism incrMulti already uses in production.
This commit is contained in:
@@ -120,6 +120,75 @@ func (s *redisStore) incrMulti(entries []counterIncr) ([]int64, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// incrAndGetMulti implements counterStore: every entry's INCRBY+EXPIRE pair
|
||||
// AND every read key's GET, sent as ONE pipeline (perf review round 3,
|
||||
// 2026-08-22) — 2*len(entries)+len(reads) commands, one round trip
|
||||
// regardless of how many of each there are, fusing checkAndCount's own
|
||||
// request-counter increments with its token/cost budget reads (limits.go's
|
||||
// counterStore doc comment explains why that fusion is safe). Reply layout
|
||||
// mirrors incrMulti's own: entry i's INCRBY reply sits at reply index 2*i
|
||||
// (EXPIRE's reply at 2*i+1 is not otherwise inspected, same reasoning as
|
||||
// incrMulti's own doc comment), followed by one GET reply per read key, in
|
||||
// order — a missing read key (RESP null bulk) maps to 0, matching
|
||||
// getBatch's own convention. The same at-least-once semantics as incrMulti
|
||||
// (see its own doc comment) apply to the INCRBY half of this call; the GET
|
||||
// half is read-only and carries no such caveat.
|
||||
func (s *redisStore) incrAndGetMulti(entries []counterIncr, reads []string) ([]int64, []int64, error) {
|
||||
cmds := make([][]string, 0, len(entries)*2+len(reads))
|
||||
for _, e := range entries {
|
||||
cmds = append(cmds,
|
||||
[]string{"INCRBY", e.key, strconv.FormatInt(e.delta, 10)},
|
||||
[]string{"EXPIRE", e.key, strconv.FormatInt(ttlToSeconds(e.ttl), 10)},
|
||||
)
|
||||
}
|
||||
for _, k := range reads {
|
||||
cmds = append(cmds, []string{"GET", k})
|
||||
}
|
||||
|
||||
replies, err := s.client.pipeline(cmds)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti: %w", err)
|
||||
}
|
||||
if len(replies) != len(cmds) {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti: expected %d replies, got %d", len(cmds), len(replies))
|
||||
}
|
||||
|
||||
incrVals := make([]int64, len(entries))
|
||||
for i, e := range entries {
|
||||
reply := replies[i*2]
|
||||
if re, ok := reply.(error); ok {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti %q: INCRBY failed: %w", e.key, re)
|
||||
}
|
||||
v, ok := reply.(int64)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti %q: unexpected INCRBY reply type %T", e.key, reply)
|
||||
}
|
||||
incrVals[i] = v
|
||||
}
|
||||
|
||||
base := len(entries) * 2
|
||||
readVals := make([]int64, len(reads))
|
||||
for i, k := range reads {
|
||||
reply := replies[base+i]
|
||||
if reply == nil {
|
||||
continue // missing key -> 0, matching getBatch's own convention
|
||||
}
|
||||
if re, ok := reply.(error); ok {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti %q: GET failed: %w", k, re)
|
||||
}
|
||||
b, ok := reply.([]byte)
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti %q: unexpected GET reply type %T", k, reply)
|
||||
}
|
||||
n, perr := strconv.ParseInt(string(b), 10, 64)
|
||||
if perr != nil {
|
||||
return nil, nil, fmt.Errorf("redisStore: incrAndGetMulti %q: non-integer value %q: %w", k, b, perr)
|
||||
}
|
||||
readVals[i] = n
|
||||
}
|
||||
return incrVals, readVals, nil
|
||||
}
|
||||
|
||||
// getMulti implements counterStore: one pipelined GET per key
|
||||
// (respClient.getBatch), a single round trip regardless of len(keys) —
|
||||
// used by limiter.currentUsage (admin dashboard, spec §4) to read a
|
||||
|
||||
Reference in New Issue
Block a user