Compare commits

..
4 Commits
Author SHA1 Message Date
github-actions[bot]andGitHub 523fc6222d chore(deps): update dependencies
Automated dependency update. All tests passed.
2026-10-29 03:04:49 +00:00
lukaszraczylo 9658f09521 fix(cache): count keys with SCAN not KEYS
CountQueries and CountQueriesWithPattern ran KEYS. With an empty
prefix, KEYS * builds a reply of every key on the server. The stats
endpoint calls it on each request. On a shared Dragonfly with about
1M keys, this raised memory on the server.

Count with a cursor SCAN instead. The count does not hold the key
list in memory. A test checks that no KEYS command is sent.
2026-10-28 22:19:10 +00:00
github-actions[bot]andGitHub 8456860dbe chore(deps): update dependencies
Automated dependency update. All tests passed.
2026-10-27 03:06:19 +00:00
github-actions[bot]andGitHub 5fa78e0fe8 chore(deps): update module github.com/gofiber/contrib/v3/websocket to v1.2.9
Automated dependency update. All tests passed.
2026-10-25 03:04:34 +00:00
4 changed files with 125 additions and 25 deletions
+26 -8
View File
@@ -32,6 +32,9 @@ func escapeGlobPattern(s string) string {
return globEscaper.Replace(s)
}
// scanCountBatch is the SCAN COUNT hint used when counting keys.
const scanCountBatch = 1000
type RedisConfig struct {
ctx context.Context
client *redis.Client
@@ -163,12 +166,28 @@ func (c *RedisConfig) Clear() error {
// literally instead of being interpreted as a wildcard, which would
// otherwise undercount or overcount keys in admin-dashboard stats.
func (c *RedisConfig) CountQueries() (int64, error) {
pattern := escapeGlobPattern(c.prefix) + "*"
keys, err := c.client.Keys(c.ctx, pattern).Result()
if err != nil {
return 0, err
return c.scanCount(escapeGlobPattern(c.prefix) + "*")
}
// scanCount counts keys matching pattern with a cursor SCAN. KEYS builds the
// whole reply in memory on the server and blocks it, which spiked a shared
// Dragonfly to its memory limit when the prefix is empty.
func (c *RedisConfig) scanCount(pattern string) (int64, error) {
var (
cursor uint64
total int64
)
for {
keys, next, err := c.client.Scan(c.ctx, cursor, pattern, scanCountBatch).Result()
if err != nil {
return 0, fmt.Errorf("redis: scan at cursor %d: %w", cursor, err)
}
total += int64(len(keys))
cursor = next
if cursor == 0 {
return total, nil
}
}
return int64(len(keys)), nil
}
// CountQueriesWithPattern returns the number of keys owned by this cache
@@ -177,12 +196,11 @@ func (c *RedisConfig) CountQueries() (int64, error) {
// itself is left as given so a caller can still use glob metacharacters
// intentionally in the suffix it supplies.
func (c *RedisConfig) CountQueriesWithPattern(pattern string) (int, error) {
fullPattern := escapeGlobPattern(c.prefix) + pattern
keys, err := c.client.Keys(c.ctx, fullPattern).Result()
total, err := c.scanCount(escapeGlobPattern(c.prefix) + pattern)
if err != nil {
return 0, err
}
return len(keys), nil
return int(total), nil
}
// GetMemoryUsage returns an approximation of memory usage for Redis.
+82
View File
@@ -0,0 +1,82 @@
package libpack_cache_redis
import (
"context"
"fmt"
"net"
"strings"
"sync"
"testing"
"github.com/alicebob/miniredis/v2"
redis "github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type commandRecorder struct {
mu sync.Mutex
seen []string
}
func (r *commandRecorder) DialHook(next redis.DialHook) redis.DialHook {
return func(ctx context.Context, network, addr string) (net.Conn, error) {
return next(ctx, network, addr)
}
}
func (r *commandRecorder) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
r.mu.Lock()
r.seen = append(r.seen, strings.ToLower(cmd.Name()))
r.mu.Unlock()
return next(ctx, cmd)
}
}
func (r *commandRecorder) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return next
}
func (r *commandRecorder) count(name string) int {
r.mu.Lock()
defer r.mu.Unlock()
n := 0
for _, s := range r.seen {
if s == name {
n++
}
}
return n
}
// TestCount_UsesScanNotKeys guards against reintroducing KEYS, which builds
// the whole key list server-side and spiked a shared Dragonfly to its limit.
func TestCount_UsesScanNotKeys(t *testing.T) {
t.Parallel()
s, err := miniredis.Run()
require.NoError(t, err)
t.Cleanup(s.Close)
rc, err := New(&RedisClientConfig{RedisServer: s.Addr()})
require.NoError(t, err)
const total = 2500
for i := 0; i < total; i++ {
require.NoError(t, rc.Set(fmt.Sprintf("k%d", i), []byte("v"), 0))
}
rec := &commandRecorder{}
rc.client.AddHook(rec)
count, err := rc.CountQueries()
require.NoError(t, err)
assert.Equal(t, int64(total), count)
patternCount, err := rc.CountQueriesWithPattern("k1*")
require.NoError(t, err)
assert.Equal(t, 1111, patternCount, "k1, k10-k19, k100-k199, k1000-k1999")
assert.Zero(t, rec.count("keys"), "counting must not issue KEYS")
assert.GreaterOrEqual(t, rec.count("scan"), 2, "counting must issue SCAN")
}
+5 -5
View File
@@ -10,12 +10,12 @@ require (
github.com/alicebob/miniredis/v2 v2.40.0
github.com/avast/retry-go/v5 v5.0.0
github.com/goccy/go-json v0.11.2
github.com/gofiber/contrib/v3/websocket v1.2.8
github.com/gofiber/contrib/v3/websocket v1.2.9
github.com/gofiber/fiber/v3 v3.6.1
github.com/gofrs/flock v0.13.1
github.com/golang-jwt/jwt/v5 v5.3.1
github.com/google/uuid v1.6.0
github.com/gookit/goutil v0.8.3
github.com/gookit/goutil v0.8.4
github.com/gorilla/websocket v1.5.3
github.com/graphql-go/graphql v0.8.1
github.com/jackc/pgx/v5 v5.11.0
@@ -26,7 +26,7 @@ require (
github.com/redis/go-redis/v9 v9.23.0
github.com/sony/gobreaker/v2 v2.4.0
github.com/stretchr/testify v1.12.1
github.com/valyala/fasthttp v1.76.0
github.com/valyala/fasthttp v1.77.0
go.opentelemetry.io/otel v1.47.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
@@ -44,7 +44,7 @@ require (
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gofiber/schema v1.9.0 // indirect
github.com/gofiber/utils/v2 v2.6.2 // indirect
github.com/gofiber/utils/v2 v2.6.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.32.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
@@ -52,7 +52,7 @@ require (
github.com/klauspost/compress v1.20.3 // indirect
github.com/mattn/go-colorable v0.1.16 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/molecule-man/go-brrr v1.3.0 // indirect
github.com/molecule-man/go-brrr v1.3.1 // indirect
github.com/philhofer/fwd v1.2.1 // indirect
github.com/savsgio/gotils v0.0.0-20250924091648-bce9a52d7761 // indirect
github.com/tinylib/msgp v1.6.5 // indirect
+12 -12
View File
@@ -32,14 +32,14 @@ github.com/goccy/go-json v0.11.2 h1:jdZv93Tt4ioR8yW1CoNsvSxrcZlCXAUU1aZXN7gpXUA=
github.com/goccy/go-json v0.11.2/go.mod h1:3NdmfEkZlB7YI5UFw/qdFKq8XN1aiWR0YyRPWZNQltY=
github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms=
github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE=
github.com/gofiber/contrib/v3/websocket v1.2.8 h1:uG/A/MGzbnXJHRPtKMDWG5LmOLodefhRMrhl9BPyKc8=
github.com/gofiber/contrib/v3/websocket v1.2.8/go.mod h1:E9TgR1WFzxsd9kIOwqyZHQx9UYKMn1ZyqxV88XITKvQ=
github.com/gofiber/contrib/v3/websocket v1.2.9 h1:DgZgxWK5Yua3g5+I4TmoA0RRx6MILIaqooSGKMgjWZE=
github.com/gofiber/contrib/v3/websocket v1.2.9/go.mod h1:+unMmdAcLfIywqYDaH8eTynn503n7Lf4Cr4kXM5ghO0=
github.com/gofiber/fiber/v3 v3.6.1 h1:wYjbgqpJm6VErIVLSGzGpX7lLZyTNgElsj8nQ1i+iFo=
github.com/gofiber/fiber/v3 v3.6.1/go.mod h1:+nxjBal3gw0EPf3gqK5EmtxaxTL01+zbSTagFletlDI=
github.com/gofiber/schema v1.9.0 h1:Dahwvth6qy1ELuHUkxk4ca+pW9zTBlak+Sw7pWJ7Iek=
github.com/gofiber/schema v1.9.0/go.mod h1:M0dV/AiPKfzHazzSkCPqragxubCvz8pP2GjJmFBsG+w=
github.com/gofiber/utils/v2 v2.6.2 h1:+tX8EPc/WO4f8DiV1W3gGWPJvGdHFW3qLIR2ws06At0=
github.com/gofiber/utils/v2 v2.6.2/go.mod h1:8ey6lqPABeSEhqoxsh/5dMyUOMHB/rkPagXlKWvL67o=
github.com/gofiber/utils/v2 v2.6.3 h1:VlnOB89rFnonlXoYOkESOkXtIkAw4CBBwpBHu+/a/YQ=
github.com/gofiber/utils/v2 v2.6.3/go.mod h1:Mq3OXwyBPxORb3jPafy6Swki4/TJJW9TDPHeerXkXpg=
github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo=
github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
@@ -50,8 +50,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gookit/goutil v0.8.3 h1:CpVihnR0uDlruMRnai9PYaSKZ7TTSgOif6Ns17b79tA=
github.com/gookit/goutil v0.8.3/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gookit/goutil v0.8.4 h1:LU93xrWsR+csbPMLI65pavJ/qD65g8pWeRaTdqbmHA0=
github.com/gookit/goutil v0.8.4/go.mod h1:vJS9HXctYTCLtCsZot5L5xF+O1oR17cDYO9R0HxBmnU=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/graphql-go/graphql v0.8.1 h1:p7/Ou/WpmulocJeEx7wjQy611rtXGQaAcXGqanuMMgc=
@@ -82,8 +82,8 @@ github.com/mattn/go-colorable v0.1.16 h1:+2YjeAe7xP1OQbZW1jERPWWCDLlNqFNCvjiqGiA
github.com/mattn/go-colorable v0.1.16/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/molecule-man/go-brrr v1.3.0 h1:KEGPkUk9PYFw5+Sv/evCOSU5OmMDXCGFqQ9YjRB7GeU=
github.com/molecule-man/go-brrr v1.3.0/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4=
github.com/molecule-man/go-brrr v1.3.1 h1:EwaJUzRIlp/dFmIpzq4J17CA5Zi5OHo2vOGmu09hF1w=
github.com/molecule-man/go-brrr v1.3.1/go.mod h1:7ybW6/7gA3oKY45jOfVNjSJDtrr6ea4tzbsTkjmQDC4=
github.com/philhofer/fwd v1.2.1 h1:UtpS73q8bFgy/yH3gCq8LSNvj5gv/FowNOiWZa7uKwQ=
github.com/philhofer/fwd v1.2.1/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -93,8 +93,8 @@ github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBP
github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/savsgio/gotils v0.0.0-20250924091648-bce9a52d7761 h1:McifyVxygw1d67y6vxUqls2D46J8W9nrki9c8c0eVvE=
github.com/savsgio/gotils v0.0.0-20250924091648-bce9a52d7761/go.mod h1:Vi9gvHvTw4yCUHIznFl5TPULS7aXwgaTByGeBY75Wko=
github.com/shamaton/msgpack/v3 v3.2.3 h1:0r0aZtrXDFOPpwWJTEOpFq+Ih/JmMLm0pqLN5mSilrw=
github.com/shamaton/msgpack/v3 v3.2.3/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo=
github.com/shamaton/msgpack/v3 v3.2.6 h1:i8I++0ehiIO8yK4+ZU6j5MIWFz5e8J6EeJvdM/qeyTw=
github.com/shamaton/msgpack/v3 v3.2.6/go.mod h1:sgBYvEiyz8JR1NC3yGRoPVME9xXovpnh3l/plW1nfRo=
github.com/sony/gobreaker/v2 v2.4.0 h1:g2KJRW1Ubty3+ZOcSEUN7K+REQJdN6yo6XvaML+jptg=
github.com/sony/gobreaker/v2 v2.4.0/go.mod h1:pTyFJgcZ3h2tdQVLZZruK2C0eoFL1fb/G83wK1ZQl+s=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -106,8 +106,8 @@ github.com/tinylib/msgp v1.6.5 h1:iAH6XTP7BpzErjBZlujJQXxA+GmRi2YWs6M4KdLtNbE=
github.com/tinylib/msgp v1.6.5/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.76.0 h1:QitrT4foDQuqvUnr/hzmQUDbYzV12Zz4vkLOlipU4Cw=
github.com/valyala/fasthttp v1.76.0/go.mod h1:m8oTPC8riDDky07invpl+Gz3RixnAN5BbwyUuVB5H2o=
github.com/valyala/fasthttp v1.77.0 h1:6mxBJdHabRmU4Cmw79Uqhgvjzsp6pjOaQk2t8COflzo=
github.com/valyala/fasthttp v1.77.0/go.mod h1:m8oTPC8riDDky07invpl+Gz3RixnAN5BbwyUuVB5H2o=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=