increase error handling and mutex encapsulation (#12)

* increase error handling and mutex encapsulation

* undo method rename for now

* set cant return error

---------

Co-authored-by: Chris Clayton <chris.clayton@contino.io>

.github/workflows/pr.yaml

@@ -43,6 +43,7 @@ jobs:
     name: "Unit testing"
     # needs: [prepare]
     runs-on: ubuntu-latest
+    container: golang:1
     # container: github/super-linter:v4
     needs: [prepare]
 
@@ -57,6 +58,9 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+        ports:
+          # Maps the container port to the host machine
+          - 6379:6379
 
     steps:
       - name: Checkout repository
@@ -70,12 +74,16 @@ jobs:
 
       - name: Install dependencies
         run: |
+          apt-get update
+          apt-get install ca-certificates make -y
+          update-ca-certificates
           go mod tidy
 
       - name: Run unit tests
         env:
           REDIS_HOST: redis
           REDIS_PORT: 6379
+          REDIS_SERVER: "redis:6379"
         run: |
           export REDIS_SERVER="$REDIS_HOST:$REDIS_PORT"
           CI_RUN=${CI} make test

Dockerfile

@@ -2,6 +2,7 @@ FROM gcr.io/distroless/base-debian12:nonroot
 WORKDIR /go/src/app
 ARG TARGETARCH
 ARG TARGETOS
+# silly workaround for distroless image as no chmod is available
+COPY --chmod=777 --chown=nonroot:nonroot static/app /go/src/app
 ADD dist/bot-$TARGETOS-$TARGETARCH /go/src/app/graphql-proxy
-ADD static/default-ratelimit.json /app/ratelimit.json
 ENTRYPOINT ["/go/src/app/graphql-proxy"]

README.md

@@ -16,6 +16,8 @@ This project is in active use by [telegram-bot.app](https://telegram-bot.app), a
   - [Speed](#speed)
     - [Caching](#caching)
     - [Read-only endpoint](#read-only-endpoint)
+  - [Maintenance](#maintenance)
+    - [Hasura event cleaner](#hasura-event-cleaner)
   - [Security](#security)
     - [Role-based rate limiting](#role-based-rate-limiting)
     - [Read-only mode](#read-only-mode)
@@ -23,6 +25,7 @@ This project is in active use by [telegram-bot.app](https://telegram-bot.app), a
     - [Blocking introspection](#blocking-introspection)
   - [API endpoints](#api-endpoints)
     - [Ban or unban the user](#ban-or-unban-the-user)
+    - [Cache operations](#cache-operations)
   - [General](#general)
     - [Metrics which matter](#metrics-which-matter)
     - [Healthcheck](#healthcheck)
@@ -100,6 +103,7 @@ In this case, both proxy and websockets will be available under the `/v1/graphql
 | security   | Blocking mutations in read-only mode                                  |
 | security   | Allow access only to listed URLs                                      |
 | security   | Ban / unban specific user from accessing the application              |
+| maintenance | Hasura event cleaner                                                 |
 
 
 ### Configuration
@@ -121,6 +125,10 @@ You can still use the non-prefixed environment variables in the spirit of the ba
 | `ROLE_RATE_LIMIT`         | Enable request rate limiting based on role| `false`                   |
 | `ENABLE_GLOBAL_CACHE`     | Enable the cache                        | `false`                    |
 | `CACHE_TTL`               | The cache TTL                           | `60`                       |
+| `ENABLE_REDIS_CACHE`      | Enable distributed Redis cache          | `false`                    |
+| `CACHE_REDIS_URL`         | URL to redis server / cluster endpoint  | `localhost:6379`           |
+| `CACHE_REDIS_PASSWORD`    | Redis connection password               | ``                         |
+| `CACHE_REDIS_DB`          | Redis DB id                             | `0`                        |
 | `LOG_LEVEL`               | The log level                           | `info`                     |
 | `BLOCK_SCHEMA_INTROSPECTION`| Blocks the schema introspection       | `false`                    |
 | `ALLOWED_INTROSPECTION`  | Allow only certain queries in introspection | ``                  |
@@ -133,13 +141,16 @@ You can still use the non-prefixed environment variables in the spirit of the ba
 | `PROXIED_CLIENT_TIMEOUT` | The timeout for the proxied client in seconds     | `120`                      |
 | `PURGE_METRICS_ON_CRAWL` | Purge metrics on each /metrics crawl    | `false`                      |
 | `PURGE_METRICS_ON_TIMER` | Purge metrics every x seconds. `0` - disabled | `0`                      |
+| `HASURA_EVENT_CLEANER`   | Enable the hasura event cleaner          | `false`                    |
+| `HASURA_EVENT_CLEANER_OLDER_THAN` | The interval for the hasura event cleaner (in days) | `1`                  |
+| `HASURA_EVENT_METADATA_DB` | URL to the hasura metadata database    | `postgresql://localhost:5432/hasura` |
 
 ### Speed
 
 #### Caching
 
 The cache engine is enabled in the background by default, using no additional resources.
-You can then start using the cache by setting the `ENABLE_GLOBAL_CACHE` environment variable to `true` - which will enable the cache for all queries without introspection. You can leave the global cache disabled and enable the cache for specific queries by adding the `@cached` directive to the query.
+You can then start using the cache by setting the `ENABLE_GLOBAL_CACHE` or `ENABLE_REDIS_CACHE` environment variable to `true` - which will enable the cache for all queries without introspection. You can leave the global cache disabled and enable the cache for specific queries by adding the `@cached` directive to the query.
 
 In the case of the `@cached` you can add additional parameters to the directive which will set the cache for specific queries to the provided time.
 For example, `query MyCachedQuery @cached(ttl: 90) ....` will set the cache for the query to 90 seconds.
@@ -158,6 +169,7 @@ query MyProducts @cached(refresh: true) {
 ```
 
 Since version `0.5.30` the cache is gzipped in the memory, which should optimise the memory usage quite significantly.
+Since version `0.15.48` the you can also use the distributed Redis cache.
 
 #### Read-only endpoint
 
@@ -165,6 +177,20 @@ You can now specify the read-only GraphQL endpoint by setting the `HOST_GRAPHQL_
 
 You can check out the [example of combined deployment with RW and read-only hasura](static/kubernetes-single-deployment-with-ro.yaml).
 
+### Maintenance
+
+#### Hasura event cleaner
+
+When enabled via `HASURA_EVENT_CLEANER=true` - proxy needs to have a direct access to the database to execute simple delete queries on schedule. You can specify number of days the logs should be kept for using `HASURA_EVENT_CLEANER_OLDER_THAN`, for example `HASURA_EVENT_CLEANER_OLDER_THAN=14` will keep 14 days of event execution logs. Ticker managing the cleaner routine will be executed every hour.
+
+Following tables are being cleaned:
+- `hdb_catalog.event_invocation_logs`
+- `hdb_catalog.event_log`
+- `hdb_catalog.hdb_action_log`
+- `hdb_catalog.hdb_cron_event_invocation_logs`
+- `hdb_catalog.hdb_scheduled_event_invocation_logs`
+
+
 ### Security
 
 #### Role-based rate limiting
@@ -236,6 +262,11 @@ To do so - you need to enable the api by setting env variable `ENABLE_API=true`
 * `POST /api/user-ban` - ban the user from accessing the application
 * `POST /api/user-unban` - unban the user from accessing the application
 
+#### Cache operations
+
+* `POST /api/cache-clear` - clear the cache
+* `GET /api/cache-stats` - get the cache statistics ( hits, misses, size )
+
 Both endpoints require the `user_id` parameter to be present in the request body and allow you to provide the reason for the ban.
 
 Example request:
@@ -283,4 +314,7 @@ graphql_proxy_executed_query{user_id="-",op_type="query",op_name="checkIfSpamAIR
 graphql_proxy_requests_failed 324
 graphql_proxy_requests_skipped 0
 graphql_proxy_requests_succesful 454823
+graphql_proxy_cache_hit{microservice="graphql_proxy",pod="hasura-w-proxy-internal-6b5f4b4bbb-9xwfc"} 7
+graphql_proxy_cache_hit{pod="hasura-w-proxy-internal-6b5f4b4bbb-9xwfc",microservice="graphql_proxy"} 1
+graphql_proxy_cache_miss{microservice="graphql_proxy",pod="hasura-w-proxy-internal-6b5f4b4bbb-9xwfc"} 23
 ```

api.go

@@ -23,6 +23,8 @@ func enableApi() {
 		api := apiserver.Group("/api")
 		api.Post("/user-ban", apiBanUser)
 		api.Post("/user-unban", apiUnbanUser)
+		api.Post("/cache-clear", apiClearCache)
+		api.Get("/cache-stats", apiCacheStats)
 
 		go periodicallyReloadBannedUsers()
 		err := apiserver.Listen(fmt.Sprintf(":%d", cfg.Server.ApiPort))
@@ -50,6 +52,25 @@ func checkIfUserIsBanned(c *fiber.Ctx, userID string) bool {
 	return found
 }
 
+func apiClearCache(c *fiber.Ctx) error {
+	cfg.Logger.Debug("Clearing cache via API", nil)
+	cacheClear()
+	cfg.Logger.Info("Cache cleared via API", nil)
+	c.Status(200).SendString("OK: cache cleared")
+	return nil
+}
+
+func apiCacheStats(c *fiber.Ctx) error {
+	stats := getCacheStats()
+	cfg.Logger.Debug("Getting cache stats via API", map[string]interface{}{"stats": stats})
+	err := c.JSON(stats)
+	if err != nil {
+		cfg.Logger.Error("Can't marshal cache stats", map[string]interface{}{"error": err.Error()})
+		return err
+	}
+	return nil
+}
+
 type apiBanUserRequest struct {
 	UserID string `json:"user_id"`
 	Reason string `json:"reason"`
@@ -111,6 +132,12 @@ func loadBannedUsers() {
 			cfg.Logger.Error("Can't create the file", map[string]interface{}{"error": err.Error()})
 			return
 		}
+		// write empty json to the file
+		err = os.WriteFile(cfg.Api.BannedUsersFile, []byte("{}"), 0644)
+		if err != nil {
+			cfg.Logger.Error("Can't write to the file", map[string]interface{}{"error": err.Error()})
+			return
+		}
 	}
 
 	fileLock := flock.New(fmt.Sprintf("%s.lock", cfg.Api.BannedUsersFile))

cache.go

@@ -5,7 +5,26 @@ import (
 
 	fiber "github.com/gofiber/fiber/v2"
 	"github.com/gookit/goutil/strutil"
-	libpack_cache "github.com/lukaszraczylo/graphql-monitoring-proxy/cache"
+	libpack_cache "github.com/lukaszraczylo/graphql-monitoring-proxy/cache/memory"
+	libpack_redis "github.com/lukaszraczylo/graphql-monitoring-proxy/cache/redis"
+)
+
+type CacheStats struct {
+	CachedQueries int `json:"cached_queries"`
+	CacheHits     int `json:"cache_hits"`
+	CacheMisses   int `json:"cache_misses"`
+}
+
+type CacheClient interface {
+	Set(key string, value []byte, ttl time.Duration)
+	Get(key string) ([]byte, bool)
+	Delete(key string)
+	Clear()
+	CountQueries() int
+}
+
+var (
+	cacheStats *CacheStats
 )
 
 func calculateHash(c *fiber.Ctx) string {
@@ -13,17 +32,64 @@ func calculateHash(c *fiber.Ctx) string {
 }
 
 func enableCache() {
-	cfg.Cache.CacheClient = libpack_cache.New(time.Duration(cfg.Cache.CacheTTL) * time.Second)
+	cacheStats = &CacheStats{}
+	if shouldUseRedisCache() {
+		cfg.Logger.Info("Using Redis cache", nil)
+		cfg.Cache.Client = libpack_redis.NewClient(&libpack_redis.RedisClientConfig{
+			RedisDB:       cfg.Cache.CacheRedisDB,
+			RedisServer:   cfg.Cache.CacheRedisURL,
+			RedisPassword: cfg.Cache.CacheRedisPassword,
+		})
+	} else {
+		cfg.Logger.Info("Using in-memory cache", nil)
+		cfg.Cache.Client = libpack_cache.New(time.Duration(cfg.Cache.CacheTTL) * time.Second)
+	}
 }
 
 func cacheLookup(hash string) []byte {
-	obj, found := cfg.Cache.CacheClient.Get(hash)
+	obj, found := cfg.Cache.Client.Get(hash)
 	if found {
+		cacheStats.CacheHits++
 		return obj
 	}
+	cacheStats.CacheMisses++
 	return nil
 }
 
 func cacheDelete(hash string) {
-	cfg.Cache.CacheClient.Delete(hash)
+	cfg.Logger.Debug("Deleting data from cache", map[string]interface{}{"hash": hash})
+	cacheStats.CachedQueries--
+	cfg.Cache.Client.Delete(hash)
+}
+
+func cacheStore(hash string, data []byte) {
+	cfg.Logger.Debug("Storing data in cache", map[string]interface{}{"hash": hash})
+	cacheStats.CachedQueries++
+	cfg.Cache.Client.Set(hash, data, time.Duration(cfg.Cache.CacheTTL)*time.Second)
+}
+
+func cacheStoreWithTTL(hash string, data []byte, ttl time.Duration) {
+	cfg.Logger.Debug("Storing data in cache with TTL", map[string]interface{}{"hash": hash, "ttl": ttl})
+	cacheStats.CachedQueries++
+	cfg.Cache.Client.Set(hash, data, ttl)
+}
+
+func cacheGetQueries() int {
+	cfg.Logger.Debug("Counting cache queries", nil)
+	return cfg.Cache.Client.CountQueries()
+}
+
+func cacheClear() {
+	cfg.Cache.Client.Clear()
+	cacheStats = &CacheStats{}
+}
+
+func getCacheStats() *CacheStats {
+	cfg.Logger.Debug("Getting cache stats", nil)
+	cacheStats.CachedQueries = cacheGetQueries()
+	return cacheStats
+}
+
+func shouldUseRedisCache() bool {
+	return cfg.Cache.CacheRedisEnable
 }

cache/cache.go

@@ -1,145 +0,0 @@
-package libpack_cache
-
-import (
-	"bytes"
-	"compress/gzip"
-	"io"
-	"sync"
-	"time"
-)
-
-type CacheEntry struct {
-	ExpiresAt time.Time
-	Value     []byte
-}
-
-type Cache struct {
-	bytePool  sync.Pool
-	entries   sync.Map
-	globalTTL time.Duration
-	sync.RWMutex
-}
-
-func New(globalTTL time.Duration) *Cache {
-	cache := &Cache{
-		globalTTL: globalTTL,
-	}
-
-	// Initialize the byte pool.
-	cache.bytePool.New = func() interface{} {
-		return make([]byte, 0)
-	}
-
-	// Start the cache cleanup.
-	go cache.cleanupRoutine(globalTTL)
-	return cache
-}
-
-func (c *Cache) cleanupRoutine(globalTTL time.Duration) {
-	ticker := time.NewTicker(globalTTL / 2)
-	defer ticker.Stop()
-
-	for range ticker.C {
-		c.CleanExpiredEntries()
-	}
-}
-
-func (c *Cache) Set(key string, value []byte, ttl time.Duration) {
-	c.Lock()
-	defer c.Unlock()
-	expiresAt := time.Now().Add(ttl)
-
-	compressedValue, err := c.compress(value)
-	if err != nil {
-		return
-	}
-
-	// Get a byte slice from the pool and ensure it's properly sized.
-	b := c.bytePool.Get().([]byte)
-	if cap(b) < len(compressedValue) {
-		b = make([]byte, len(compressedValue))
-	} else {
-		b = b[:len(compressedValue)]
-	}
-
-	copy(b, compressedValue)
-
-	entry := CacheEntry{
-		Value:     b,
-		ExpiresAt: expiresAt,
-	}
-	c.entries.Store(key, entry)
-}
-
-func (c *Cache) Get(key string) ([]byte, bool) {
-	c.RLock()
-	defer c.RUnlock()
-
-	entry, ok := c.entries.Load(key)
-	if !ok || entry.(CacheEntry).ExpiresAt.Before(time.Now()) {
-		return nil, false
-	}
-	compressedValue := entry.(CacheEntry).Value
-	value, err := c.decompress(compressedValue)
-	if err != nil {
-		return nil, false
-	}
-
-	return value, true
-}
-
-func (c *Cache) Delete(key string) {
-	c.Lock()
-	defer c.Unlock()
-
-	entry, ok := c.entries.Load(key)
-	if !ok {
-		return
-	}
-
-	// Return the byte slice to the pool.
-	c.bytePool.Put(entry.(CacheEntry).Value)
-
-	// Delete the entry from the cache.
-	c.entries.Delete(key)
-}
-
-func (c *Cache) CleanExpiredEntries() {
-	now := time.Now()
-	c.entries.Range(func(key, value interface{}) bool {
-		entry := value.(CacheEntry)
-		if entry.ExpiresAt.Before(now) {
-			// Return the byte slice to the pool.
-			c.bytePool.Put(entry.Value)
-
-			// Delete the entry from the cache.
-			c.entries.Delete(key)
-		}
-
-		// Return true to continue iterating over the map.
-		return true
-	})
-}
-
-func (c *Cache) compress(data []byte) ([]byte, error) {
-	var buf bytes.Buffer
-	w := gzip.NewWriter(&buf)
-	_, err := w.Write(data)
-	if err != nil {
-		return nil, err
-	}
-	err = w.Close()
-	if err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func (c *Cache) decompress(data []byte) ([]byte, error) {
-	r, err := gzip.NewReader(bytes.NewBuffer(data))
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-	return io.ReadAll(r)
-}

cache/memory/cache.go

@@ -0,0 +1,186 @@
+package libpack_cache
+
+import (
+	"bytes"
+	"compress/gzip"
+	"io"
+	"log"
+	"sync"
+	"time"
+)
+
+type CacheEntry struct {
+	ExpiresAt time.Time
+	Value     []byte
+}
+
+type Cache struct {
+	compressPool   sync.Pool
+	decompressPool sync.Pool
+	entries        sync.Map
+	globalTTL      time.Duration
+	mu             sync.RWMutex // Added sync.RWMutex field for locking
+}
+
+func New(globalTTL time.Duration) *Cache {
+	cache := &Cache{
+		globalTTL: globalTTL,
+		compressPool: sync.Pool{
+			New: func() interface{} {
+				return gzip.NewWriter(nil)
+			},
+		},
+		decompressPool: sync.Pool{
+			New: func() interface{} {
+				r, _ := gzip.NewReader(bytes.NewReader([]byte{}))
+				return r
+			},
+		},
+	}
+
+	go cache.cleanupRoutine(globalTTL)
+	return cache
+}
+
+func (c *Cache) cleanupRoutine(globalTTL time.Duration) {
+	ticker := time.NewTicker(globalTTL / 2)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		c.CleanExpiredEntries()
+	}
+}
+
+func (c *Cache) Set(key string, value []byte, ttl time.Duration) {
+	c.lock()
+	defer c.unlock()
+
+	expiresAt := time.Now().Add(ttl)
+
+	compressedValue, err := c.compress(value)
+	if err != nil {
+		log.Printf("Error compressing value for key %s: %v", key, err)
+		return
+	}
+
+	entry := CacheEntry{
+		Value:     compressedValue,
+		ExpiresAt: expiresAt,
+	}
+	c.entries.Store(key, entry)
+}
+
+func (c *Cache) Get(key string) ([]byte, bool) {
+	c.rlock()
+	defer c.runlock()
+
+	entry, ok := c.entries.Load(key)
+	if !ok || entry.(CacheEntry).ExpiresAt.Before(time.Now()) {
+		return nil, false
+	}
+	compressedValue := entry.(CacheEntry).Value
+	value, err := c.decompress(compressedValue)
+	if err != nil {
+		log.Printf("Error decompressing value for key %s: %v", key, err)
+		return nil, false
+	}
+	return value, true
+}
+
+func (c *Cache) Delete(key string) {
+	c.lock()
+	defer c.unlock()
+
+	c.entries.Delete(key)
+}
+
+func (c *Cache) Clear() {
+	c.lock()
+	defer c.unlock()
+
+	c.entries.Range(func(key, value interface{}) bool {
+		c.entries.Delete(key)
+		return true
+	})
+}
+
+func (c *Cache) CountQueries() int {
+	c.rlock()
+	defer c.runlock()
+
+	var count int
+	c.entries.Range(func(_, _ interface{}) bool {
+		count++
+		return true
+	})
+	return count
+}
+
+func (c *Cache) compress(data []byte) ([]byte, error) {
+	w := c.compressPool.Get().(*gzip.Writer)
+	defer c.compressPool.Put(w)
+
+	var buf bytes.Buffer
+	w.Reset(&buf)
+	if _, err := w.Write(data); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func (c *Cache) decompress(data []byte) ([]byte, error) {
+	r, ok := c.decompressPool.Get().(*gzip.Reader)
+	if !ok || r == nil {
+		var err error
+		r, err = gzip.NewReader(bytes.NewReader(data))
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		if err := r.Reset(bytes.NewReader(data)); err != nil {
+			return nil, err
+		}
+	}
+	defer func() {
+		r.Close()
+		c.decompressPool.Put(r)
+	}()
+
+	decompressedData, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	return decompressedData, nil
+}
+
+func (c *Cache) CleanExpiredEntries() {
+	now := time.Now()
+	c.entries.Range(func(key, value interface{}) bool {
+		entry := value.(CacheEntry)
+		if entry.ExpiresAt.Before(now) {
+			c.entries.Delete(key)
+		}
+		return true
+	})
+}
+
+// Private methods to handle locking
+
+func (c *Cache) lock() {
+	c.mu.Lock()
+}
+
+func (c *Cache) unlock() {
+	c.mu.Unlock()
+}
+
+func (c *Cache) rlock() {
+	c.mu.RLock()
+}
+
+func (c *Cache) runlock() {
+	c.mu.RUnlock()
+}

cache/memory/cache_bench_test.go

@@ -0,0 +1,54 @@
+package libpack_cache
+
+import (
+	"testing"
+	"time"
+)
+
+// Assume that New function initializes the cache and it is defined somewhere in the libpack_cache package.
+
+func BenchmarkCacheSet(b *testing.B) {
+	cache := New(30 * time.Second) // Initializing the cache with a TTL of 30 seconds
+	key := "benchmark-key"
+	value := []byte("benchmark-value")
+
+	b.ResetTimer() // Reset the timer to exclude the setup time from the benchmark
+
+	for i := 0; i < b.N; i++ {
+		cache.Set(key, value, 5*time.Second)
+	}
+}
+
+func BenchmarkCacheGet(b *testing.B) {
+	cache := New(30 * time.Second) // Initializing the cache
+	key := "benchmark-key"
+	value := []byte("benchmark-value")
+	cache.Set(key, value, 5*time.Second) // Pre-set a value to retrieve
+
+	b.ResetTimer() // Start timing
+
+	for i := 0; i < b.N; i++ {
+		_, _ = cache.Get(key)
+	}
+}
+
+func BenchmarkCacheExpire(b *testing.B) {
+	key := "benchmark-expire-key"
+	value := []byte("benchmark-value")
+	ttl := 5 * time.Millisecond // Setting a short TTL for quick expiration
+
+	for i := 0; i < b.N; i++ {
+		cache := New(30 * time.Second)
+		cache.Set(key, value, ttl)
+		time.Sleep(ttl) // Wait for the key to expire
+		_, _ = cache.Get(key)
+	}
+}
+
+func BenchmarkCacheStats(b *testing.B) {
+	cache := New(30 * time.Second) // Initializing the cache
+	key := "benchmark-key"
+	value := []byte("benchmark-value")
+	cache.Set(key, value, 5*time.Second) // Pre-set a value to retrieve
+	cache.Get(key)
+}

cache/memory/cache_test.go

@@ -110,39 +110,3 @@ func (suite *CacheTestSuite) Test_CacheExpire() {
 		})
 	}
 }
-
-func (suite *CacheTestSuite) Test_CacheCleanExpiredEntries() {
-	cache := New(5 * time.Second)
-	tests := []struct {
-		name        string
-		cache_value string
-		ttl         time.Duration
-	}{
-		{
-			name:        "test1",
-			cache_value: "test1-123",
-			ttl:         2 * time.Second,
-		},
-		{
-			name:        "test2",
-			cache_value: "test2-123",
-			ttl:         5 * time.Second,
-		},
-	}
-	for _, tt := range tests {
-		suite.T().Run(tt.name, func(t *testing.T) {
-			cache.Set(tt.name, []byte(tt.name), tt.ttl)
-			c, ok := cache.Get(tt.name)
-			suite.Equal(true, ok)
-			suite.Equal(tt.name, string(c))
-			time.Sleep(tt.ttl)
-			c, ok = cache.Get(tt.name)
-			suite.Equal(false, ok)
-			suite.Equal("", string(c))
-			cache.CleanExpiredEntries()
-			c, ok = cache.Get(tt.name)
-			suite.Equal(false, ok)
-			suite.Equal("", string(c))
-		})
-	}
-}

cache/redis/connection.go

@@ -0,0 +1,77 @@
+package libpack_redis
+
+import (
+	"context"
+	"time"
+
+	redis "github.com/redis/go-redis/v9"
+)
+
+var ()
+
+type RedisConfig struct {
+	client *redis.Client
+	ctx    context.Context
+}
+
+func prependKeyName(key string) string {
+	return "gmp_cache:" + key
+}
+
+type RedisClientConfig struct {
+	RedisServer   string
+	RedisPassword string
+	RedisDB       int
+}
+
+func NewClient(redisClientConfig *RedisClientConfig) *RedisConfig {
+	c := &RedisConfig{
+		client: redis.NewClient(&redis.Options{
+			Addr:     redisClientConfig.RedisServer,
+			Password: redisClientConfig.RedisPassword,
+			DB:       redisClientConfig.RedisDB,
+		}),
+		ctx: context.Background(),
+	}
+	_, err := c.client.Ping(c.ctx).Result()
+	if err != nil {
+		panic(err)
+	}
+	return c
+}
+
+func (c *RedisConfig) Set(key string, value []byte, ttl time.Duration) {
+	c.client.Set(c.ctx, prependKeyName(key), value, ttl)
+}
+
+func (c *RedisConfig) Get(key string) ([]byte, bool) {
+	val, err := c.client.Get(c.ctx, prependKeyName(key)).Result()
+	if err == redis.Nil || err != nil {
+		return nil, false
+	}
+	return []byte(val), true
+}
+
+func (c *RedisConfig) Delete(key string) {
+	c.client.Del(c.ctx, prependKeyName(key))
+}
+
+func (c *RedisConfig) Clear() {
+	c.client.FlushDB(c.ctx)
+}
+
+func (c *RedisConfig) CountQueries() int {
+	keys, err := c.client.Keys(c.ctx, prependKeyName("*")).Result()
+	if err != nil {
+		return 0
+	}
+	return len(keys)
+}
+
+func (c *RedisConfig) CountQueriesWithPattern(pattern string) int {
+	keys, err := c.client.Keys(c.ctx, prependKeyName(pattern)).Result()
+	if err != nil {
+		return 0
+	}
+	return len(keys)
+}

cache/redis/connection_test.go

@@ -0,0 +1,125 @@
+package libpack_redis
+
+import (
+	"testing"
+	"time"
+
+	"github.com/gookit/goutil/envutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type RedisConfigSuite struct {
+	suite.Suite
+	redisConfig *RedisConfig
+}
+
+func (suite *RedisConfigSuite) SetupTest() {
+	redis_server := envutil.Getenv("REDIS_SERVER", "localhost:6379")
+	suite.redisConfig = NewClient(&RedisClientConfig{
+		RedisServer:   redis_server,
+		RedisPassword: "",
+		RedisDB:       0,
+	})
+	suite.redisConfig.Delete("testkey")
+}
+
+func TestRedisConfigSuite(t *testing.T) {
+	suite.Run(t, new(RedisConfigSuite))
+}
+
+func (suite *RedisConfigSuite) TestSet() {
+	key := "testkey"
+	value := []byte("testvalue")
+	suite.redisConfig.Delete(key) // Ensure the key is deleted before the test
+
+	// Test writing a new key-value pair
+	suite.redisConfig.Set(key, value, 0)
+	storedValue, found := suite.redisConfig.Get(key)
+	assert.True(suite.T(), found)
+	assert.Equal(suite.T(), value, storedValue)
+
+	// Test overwriting an existing key-value pair
+	newValue := []byte("newvalue")
+	suite.redisConfig.Set(key, newValue, 0)
+	storedValue, found = suite.redisConfig.Get(key)
+	assert.True(suite.T(), found)
+	assert.Equal(suite.T(), newValue, storedValue)
+
+	suite.redisConfig.Delete(key) // Clean up after the test
+}
+
+func (suite *RedisConfigSuite) TestSetWithExpiry() {
+	key := "testkey"
+	value := []byte("testvalue")
+	expiry := 1 * time.Second
+	suite.redisConfig.Delete(key) // Ensure the key is deleted before the test
+
+	// Test writing a new key-value pair
+	suite.redisConfig.Set(key, value, expiry)
+	storedValue, found := suite.redisConfig.Get(key)
+	assert.True(suite.T(), found)
+	assert.Equal(suite.T(), value, storedValue)
+
+	// Test that key expires after the specified time
+	time.Sleep(2 * time.Second)
+	_, found = suite.redisConfig.Get(key)
+	assert.False(suite.T(), found)
+
+	suite.redisConfig.Delete(key) // Clean up after the test
+}
+
+func (suite *RedisConfigSuite) TestGet() {
+	key := "testkey"
+	value := []byte("testvalue")
+	suite.redisConfig.Set(key, value, 0) // Set the key-value pair
+	storedValue, found := suite.redisConfig.Get(key)
+	assert.True(suite.T(), found)
+	assert.Equal(suite.T(), value, storedValue)
+}
+
+func (suite *RedisConfigSuite) TestDeleteKey() {
+	key := "testkey"
+	value := []byte("testvalue")
+	suite.redisConfig.Set(key, value, 0) // Set the key-value pair
+	suite.redisConfig.Delete(key)
+	_, found := suite.redisConfig.Get(key)
+	assert.False(suite.T(), found)
+}
+
+func (suite *RedisConfigSuite) TestCheckIfKeyExists() {
+	ttl := time.Duration(10) * time.Second
+	key := "testkey"
+	value := []byte("testvalue")
+	suite.redisConfig.Set(key, value, ttl) // Set the key-value pair
+	_, found := suite.redisConfig.Get(key)
+	assert.True(suite.T(), found)
+
+	suite.redisConfig.Delete(key)
+	_, found = suite.redisConfig.Get(key)
+	assert.False(suite.T(), found)
+}
+
+func (suite *RedisConfigSuite) TestGetKeys() {
+	ttl := time.Duration(10) * time.Second
+	suite.redisConfig.Set("testkey1", []byte("testvalue1"), ttl)
+	suite.redisConfig.Set("testkey2", []byte("testvalue2"), ttl)
+	suite.redisConfig.Set("otherkey", []byte("othervalue"), ttl)
+
+	keys, _ := suite.redisConfig.client.Keys(suite.redisConfig.ctx, prependKeyName("testkey*")).Result()
+	expectedKeys := []string{prependKeyName("testkey1"), prependKeyName("testkey2")}
+	assert.ElementsMatch(suite.T(), expectedKeys, keys)
+
+	suite.redisConfig.client.Del(suite.redisConfig.ctx, "testkey1", "testkey2", "otherkey")
+}
+
+func (suite *RedisConfigSuite) TestGetKeysCount() {
+	ttl := time.Duration(10) * time.Second
+	suite.redisConfig.Set("testkey1", []byte("testvalue1"), ttl)
+	suite.redisConfig.Set("testkey2", []byte("testvalue2"), ttl)
+	suite.redisConfig.Set("otherkey", []byte("othervalue"), ttl)
+
+	assert.Equal(suite.T(), 2, suite.redisConfig.CountQueriesWithPattern("testkey*"))
+
+	suite.redisConfig.client.Del(suite.redisConfig.ctx, "testkey1", "testkey2", "otherkey")
+}

cache_test.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"time"
+	"github.com/gookit/goutil/envutil"
+	libpack_redis "github.com/lukaszraczylo/graphql-monitoring-proxy/cache/redis"
 )
 
-func (suite *Tests) Test_cacheLookup() {
+func (suite *Tests) Test_cacheLookupInmemory() {
 	type args struct {
 		hash string
 	}
@@ -39,7 +40,57 @@ func (suite *Tests) Test_cacheLookup() {
 	for _, tt := range tests {
 		suite.Run(tt.name, func() {
 			if tt.addCache.data != nil {
-				cfg.Cache.CacheClient.Set(tt.args.hash, tt.addCache.data, time.Duration(90*time.Second))
+				cacheStore(tt.args.hash, tt.addCache.data)
+			}
+			got := cacheLookup(tt.args.hash)
+			assert.Equal(tt.want, got, "Unexpected cache lookup result")
+		})
+	}
+}
+
+func (suite *Tests) Test_cacheLookupRedis() {
+	redis_server := envutil.Getenv("REDIS_SERVER", "localhost:6379")
+	cfg.Cache.Client = libpack_redis.NewClient(&libpack_redis.RedisClientConfig{
+		RedisServer:   redis_server,
+		RedisPassword: "",
+		RedisDB:       0,
+	})
+
+	type args struct {
+		hash string
+	}
+	tests := []struct {
+		name     string
+		args     args
+		want     []byte
+		addCache struct {
+			data []byte
+		}
+	}{
+		{
+			name: "test_non_existent",
+			args: args{
+				hash: "00000000000000000000000000000000000000",
+			},
+			want: nil,
+		},
+		{
+			name: "test_existent",
+			args: args{
+				hash: "00000000000000000000000000000000001337",
+			},
+			want: []byte("it's fine."),
+			addCache: struct {
+				data []byte
+			}{
+				data: []byte("it's fine."),
+			},
+		},
+	}
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			if tt.addCache.data != nil {
+				cacheStore(tt.args.hash, tt.addCache.data)
 			}
 			got := cacheLookup(tt.args.hash)
 			assert.Equal(tt.want, got, "Unexpected cache lookup result")

events.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+)
+
+func enableHasuraEventCleaner() {
+	if cfg.HasuraEventCleaner.Enable {
+		if cfg.HasuraEventCleaner.EventMetadataDb == "" {
+			cfg.Logger.Warning("Event metadata db URL not specified, event cleaner not active", nil)
+			return
+		}
+
+		ticker := time.NewTicker(1 * time.Hour)
+		defer ticker.Stop()
+		cfg.Logger.Info("Event cleaner enabled", map[string]interface{}{"interval_in_days": cfg.HasuraEventCleaner.ClearOlderThan})
+
+		time.Sleep(60 * time.Second) // wait for everything to start and settle down
+		cfg.Logger.Info("Initial cleanup of old events", nil)
+		cleanEvents()
+
+		for {
+			select {
+			case <-ticker.C:
+				cfg.Logger.Info("Cleaning up old events", nil)
+				cleanEvents()
+			}
+		}
+	}
+}
+
+func cleanEvents() {
+	conn, err := pgx.Connect(context.Background(), cfg.HasuraEventCleaner.EventMetadataDb)
+	if err != nil {
+		cfg.Logger.Error("Failed to connect to event metadata db", map[string]interface{}{"error": err})
+		return
+	}
+	defer conn.Close(context.Background())
+
+	delQueries := []string{
+		fmt.Sprintf("DELETE FROM hdb_catalog.event_invocation_logs WHERE created_at < now() - interval '%d days';", cfg.HasuraEventCleaner.ClearOlderThan),
+		fmt.Sprintf("DELETE FROM hdb_catalog.event_log WHERE created_at < now() - interval '%d days';", cfg.HasuraEventCleaner.ClearOlderThan),
+		fmt.Sprintf("DELETE FROM hdb_catalog.hdb_action_log WHERE created_at < NOW() - INTERVAL '%d days';", cfg.HasuraEventCleaner.ClearOlderThan),
+		fmt.Sprintf("DELETE FROM hdb_catalog.hdb_cron_event_invocation_logs WHERE created_at < NOW() - INTERVAL '%d days';", cfg.HasuraEventCleaner.ClearOlderThan),
+		fmt.Sprintf("DELETE FROM hdb_catalog.hdb_scheduled_event_invocation_logs WHERE created_at < NOW() - INTERVAL '%d days';", cfg.HasuraEventCleaner.ClearOlderThan),
+	}
+
+	for _, query := range delQueries {
+		_, err := conn.Exec(context.Background(), query)
+		if err != nil {
+			cfg.Logger.Debug("Failed to execute query", map[string]interface{}{"query": query, "error": err})
+		}
+	}
+}

go.mod

@@ -3,28 +3,33 @@ module github.com/lukaszraczylo/graphql-monitoring-proxy
 go 1.21
 
 require (
-	github.com/VictoriaMetrics/metrics v1.33.0
-	github.com/avast/retry-go/v4 v4.5.1
-	github.com/buger/jsonparser v1.1.1
-	github.com/goccy/go-json v0.10.2
-	github.com/gofiber/fiber/v2 v2.52.2
+	github.com/VictoriaMetrics/metrics v1.33.1
+	github.com/avast/retry-go/v4 v4.6.0
+	github.com/goccy/go-json v0.10.3
+	github.com/gofiber/fiber/v2 v2.52.4
 	github.com/gofrs/flock v0.8.1
 	github.com/google/uuid v1.6.0
 	github.com/gookit/goutil v0.6.15
 	github.com/graphql-go/graphql v0.8.1
+	github.com/jackc/pgx/v5 v5.6.0
 	github.com/lukaszraczylo/ask v0.0.0-20230927103145-2ff1123b4415
 	github.com/lukaszraczylo/go-ratecounter v0.1.8
-	github.com/lukaszraczylo/go-simple-graphql v1.2.9
-	github.com/rs/zerolog v1.32.0
-	github.com/stretchr/testify v1.8.4
-	github.com/valyala/fasthttp v1.52.0
+	github.com/lukaszraczylo/go-simple-graphql v1.2.14
+	github.com/redis/go-redis/v9 v9.5.3
+	github.com/rs/zerolog v1.33.0
+	github.com/stretchr/testify v1.9.0
+	github.com/valyala/fasthttp v1.54.0
 )
 
 require (
 	github.com/andybalholm/brotli v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/gookit/color v1.5.4 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
@@ -37,12 +42,12 @@ require (
 	github.com/valyala/histogram v1.2.0 // indirect
 	github.com/valyala/tcplisten v1.0.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/net v0.22.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/term v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	golang.org/x/net v0.26.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/term v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,20 +1,27 @@
-github.com/VictoriaMetrics/metrics v1.33.0 h1:EnkDEaGiL2u95t+W76GfecC/LMYpy+tFrexYzBWQIAc=
-github.com/VictoriaMetrics/metrics v1.33.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
+github.com/VictoriaMetrics/metrics v1.33.1 h1:CNV3tfm2Kpv7Y9W3ohmvqgFWPR55tV2c7M2U6OIo+UM=
+github.com/VictoriaMetrics/metrics v1.33.1/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
-github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
-github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
-github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
-github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
+github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA=
+github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofiber/fiber/v2 v2.52.2 h1:b0rYH6b06Df+4NyrbdptQL8ifuxw/Tf2DgfkZkDaxEo=
-github.com/gofiber/fiber/v2 v2.52.2/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
+github.com/gofiber/fiber/v2 v2.52.4 h1:P+T+4iK7VaqUsq2PALYEfBBo6bJZ4q3FP8cZ84EggTM=
+github.com/gofiber/fiber/v2 v2.52.4/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -25,21 +32,26 @@ github.com/gookit/goutil v0.6.15 h1:mMQ0ElojNZoyPD0eVROk5QXJPh2uKR4g06slgPDF5Jo=
 github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV3DaMDY=
 github.com/graphql-go/graphql v0.8.1 h1:p7/Ou/WpmulocJeEx7wjQy611rtXGQaAcXGqanuMMgc=
 github.com/graphql-go/graphql v0.8.1/go.mod h1:nKiHzRM0qopJEwCITUuIsxk9PlVlwIiiI8pnJEhordQ=
-github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
-github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
+github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
+github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
+github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lukaszraczylo/ask v0.0.0-20230927103145-2ff1123b4415 h1:lvI8Wlbg4PxkRcg2f10wgoaRpfN19v+YdRek3+dLtlM=
 github.com/lukaszraczylo/ask v0.0.0-20230927103145-2ff1123b4415/go.mod h1:M+UVdyqZs++xtEPrascaVmZdOMhCnxjZ2SgH+xHpR0c=
 github.com/lukaszraczylo/go-ratecounter v0.1.8 h1:ZYm6Wkn58ZAlFWRmC7PaD4oAYHWcu8/0MUDWGe3PnJQ=
 github.com/lukaszraczylo/go-ratecounter v0.1.8/go.mod h1:TqXEOCtFJStk1i0tkipprv1kiDHGon1MVUisjSTBSKM=
-github.com/lukaszraczylo/go-simple-graphql v1.2.9 h1:JKIvAw+4O8vwTv2rZKKRtn0DjLbM8XdKXZHns31Ntvc=
-github.com/lukaszraczylo/go-simple-graphql v1.2.9/go.mod h1:YOX06PIgxUyFDJZu5FFFo/9FyGTFSh9Zuld2bU8DywU=
+github.com/lukaszraczylo/go-simple-graphql v1.2.14 h1:Dth+yZ+1ialCpnslSb6UgHbXszExjDUu/I95QZbnWVU=
+github.com/lukaszraczylo/go-simple-graphql v1.2.14/go.mod h1:pSKmm9OLGoS9pjmIvhBB/fo0+LganRrL29CN3fdkRPw=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@@ -52,6 +64,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU=
+github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -59,14 +73,17 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.32.0 h1:keLypqrlIjaFsbmJOBdB/qvyF8KEtCWHwobLp5l/mQ0=
-github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.52.0 h1:wqBQpxH71XW0e2g+Og4dzQM8pk34aFYlA1Ga8db7gU0=
-github.com/valyala/fasthttp v1.52.0/go.mod h1:hf5C4QnVMkNXMspnsUlfM3WitlgYflyhHYoKol/szxQ=
+github.com/valyala/fasthttp v1.54.0 h1:cCL+ZZR3z3HPLMVfEYVUMtJqVaui0+gu7Lx63unHwS0=
+github.com/valyala/fasthttp v1.54.0/go.mod h1:6dt4/8olwq9QARP/TDuPmWyWcl4byhpvTJ4AAtcz+QM=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
@@ -75,23 +92,26 @@ github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVS
 github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

logging/logging.go

@@ -3,6 +3,7 @@ package libpack_logging
 import (
 	"io"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/gookit/goutil/envutil"
@@ -13,7 +14,21 @@ type LogConfig struct {
 	logger zerolog.Logger
 }
 
-var baseLogger zerolog.Logger
+var (
+	baseLogger zerolog.Logger
+
+	eventPool = sync.Pool{
+		New: func() interface{} {
+			return new(zerolog.Event)
+		},
+	}
+
+	fieldMapPool = sync.Pool{
+		New: func() interface{} {
+			return make(map[string]interface{})
+		},
+	}
+)
 
 func init() {
 	zerolog.TimeFieldFormat = time.RFC3339
@@ -21,10 +36,9 @@ func init() {
 	zerolog.TimestampFieldName = "timestamp"
 	zerolog.LevelFieldName = "level"
 	zerolog.LevelFatalValue = "critical"
-	baseLogger = zerolog.New(os.Stdout).With().Timestamp().Logger()
-}
 
-func NewLogger() *LogConfig {
+	baseLogger = zerolog.New(os.Stdout).With().Timestamp().Logger()
+
 	switch logLevel := envutil.Getenv("LOG_LEVEL", "info"); logLevel {
 	case "debug":
 		baseLogger = baseLogger.Level(zerolog.DebugLevel)
@@ -35,62 +49,75 @@ func NewLogger() *LogConfig {
 	default:
 		baseLogger = baseLogger.Level(zerolog.InfoLevel)
 	}
+}
 
+func NewLogger() *LogConfig {
 	return &LogConfig{logger: baseLogger}
 }
 
-func (lw *LogConfig) log(w io.Writer, level zerolog.Level, message string, v map[string]interface{}) {
-	e := lw.logger.With().Logger()
-	e = e.Output(w)
-	event := e.WithLevel(level).CallerSkipFrame(3)
-	for k, val := range v {
+func (lw *LogConfig) log(w io.Writer, level zerolog.Level, message string, fields map[string]interface{}) {
+	logger := lw.logger.Output(w)
+	event := logger.WithLevel(level).CallerSkipFrame(3)
+
+	for k, val := range fields {
 		switch v := val.(type) {
 		case string:
-			event.Str(k, v)
+			event = event.Str(k, v)
 		case int:
-			event.Int(k, v)
+			event = event.Int(k, v)
 		case float64:
-			event.Float64(k, v)
+			event = event.Float64(k, v)
 		default:
-			event.Interface(k, val)
+			event = event.Interface(k, val)
 		}
 	}
+
 	event.Msg(message)
 }
 
-func (lw *LogConfig) Debug(message string, v ...map[string]interface{}) {
-	if !lw.logger.Debug().Enabled() {
+func (lw *LogConfig) logWithLevel(level zerolog.Level, message string, fields map[string]interface{}) {
+	if lw.logger.GetLevel() > level {
 		return
 	}
-	lw.log(os.Stdout, zerolog.DebugLevel, message, mergeMaps(v))
-}
-
-func (lw *LogConfig) Info(message string, v ...map[string]interface{}) {
-	if !lw.logger.Info().Enabled() {
-		return
+	if lw.logger.GetLevel() <= level {
+		w := os.Stdout
+		if level >= zerolog.ErrorLevel {
+			w = os.Stderr
+		}
+		lw.log(w, level, message, fields)
 	}
-	lw.log(os.Stdout, zerolog.InfoLevel, message, mergeMaps(v))
 }
 
-func (lw *LogConfig) Warning(message string, v ...map[string]interface{}) {
-	lw.log(os.Stdout, zerolog.WarnLevel, message, mergeMaps(v))
+func (lw *LogConfig) Debug(message string, fields map[string]interface{}) {
+	lw.logWithLevel(zerolog.DebugLevel, message, fields)
 }
 
-func (lw *LogConfig) Error(message string, v ...map[string]interface{}) {
-	lw.log(os.Stderr, zerolog.ErrorLevel, message, mergeMaps(v))
+func (lw *LogConfig) Info(message string, fields map[string]interface{}) {
+	lw.logWithLevel(zerolog.InfoLevel, message, fields)
 }
 
-func (lw *LogConfig) Critical(message string, v ...map[string]interface{}) {
-	lw.log(os.Stderr, zerolog.FatalLevel, message, mergeMaps(v))
+func (lw *LogConfig) Warning(message string, fields map[string]interface{}) {
+	lw.logWithLevel(zerolog.WarnLevel, message, fields)
+}
+
+func (lw *LogConfig) Error(message string, fields map[string]interface{}) {
+	lw.logWithLevel(zerolog.ErrorLevel, message, fields)
+}
+
+func (lw *LogConfig) Critical(message string, fields map[string]interface{}) {
+	lw.logWithLevel(zerolog.FatalLevel, message, fields)
 	os.Exit(1)
 }
 
-func mergeMaps(maps []map[string]interface{}) map[string]interface{} {
-	result := make(map[string]interface{})
-	for _, m := range maps {
-		for k, v := range m {
-			result[k] = v
-		}
-	}
-	return result
+// Helper function to get a new fields map from the pool
+func getFieldsMap() map[string]interface{} {
+	return fieldMapPool.Get().(map[string]interface{})
+}
+
+// Helper function to put a used fields map back into the pool
+func putFieldsMap(fields map[string]interface{}) {
+	for k := range fields {
+		delete(fields, k)
+	}
+	fieldMapPool.Put(fields)
 }

logging/logging_bench_test.go

@@ -25,6 +25,7 @@ func BenchmarkInfoLog(b *testing.B) {
 	}()
 
 	testsLogger := NewLogger()
+	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		testsLogger.Info("test", map[string]interface{}{"test": "test"})
 	}

logging/logging_test.go

@@ -1,373 +0,0 @@
-package libpack_logging
-
-import (
-	"errors"
-	"io"
-	"os"
-	"reflect"
-	"testing"
-
-	"github.com/buger/jsonparser"
-	"github.com/stretchr/testify/suite"
-)
-
-type LoggingTestSuite struct {
-	suite.Suite
-}
-
-var (
-	testsLogger *LogConfig
-)
-
-type stdoutCapture struct {
-	oldStdout *os.File
-	readPipe  *os.File
-}
-
-func (sc *stdoutCapture) StartCapture() {
-	sc.oldStdout = os.Stdout
-	sc.readPipe, os.Stdout, _ = os.Pipe()
-}
-
-func (sc *stdoutCapture) StopCapture() (string, error) {
-	if sc.oldStdout == nil || sc.readPipe == nil {
-		return "", errors.New("StartCapture not called before StopCapture on Stdout")
-	}
-	os.Stdout.Close()
-	os.Stdout = sc.oldStdout
-	bytes, err := io.ReadAll(sc.readPipe)
-	if err != nil {
-		return "", err
-	}
-	return string(bytes), nil
-}
-
-type stderrCapture struct {
-	oldStderr *os.File
-	readPipe  *os.File
-}
-
-func (sc *stderrCapture) StartCapture() {
-	sc.oldStderr = os.Stderr
-	sc.readPipe, os.Stderr, _ = os.Pipe()
-}
-
-func (sc *stderrCapture) StopCapture() (string, error) {
-	if sc.oldStderr == nil || sc.readPipe == nil {
-		return "", errors.New("StartCapture not called before StopCapture on Stderr")
-	}
-	os.Stderr.Close()
-	os.Stderr = sc.oldStderr
-	bytes, err := io.ReadAll(sc.readPipe)
-	if err != nil {
-		return "", err
-	}
-	return string(bytes), nil
-}
-
-func (suite *LoggingTestSuite) SetupTest() {
-}
-
-func TestLoggingTestSuite(t *testing.T) {
-	suite.Run(t, new(LoggingTestSuite))
-}
-
-func (suite *LoggingTestSuite) TestLogConfig_AllHandlers() {
-	type args struct {
-		message string
-	}
-	tests := []struct {
-		name           string
-		args           args
-		wantLevel      string
-		wantMessage    string
-		envMinLogLevel string
-		loggerType     string
-		stdOutExpect   bool
-		stdErrExpect   bool
-	}{
-		{
-			name:       "Test log: Error",
-			loggerType: "Error",
-			args: args{
-				message: "This is a error message",
-			},
-			wantLevel:    "error",
-			wantMessage:  "This is a error message",
-			stdErrExpect: true,
-			stdOutExpect: false,
-		},
-		{
-			name:       "Test log: Warning",
-			loggerType: "Warning",
-			args: args{
-				message: "This is a warning message",
-			},
-			wantLevel:      "warn",
-			wantMessage:    "This is a warning message",
-			stdErrExpect:   false,
-			stdOutExpect:   true,
-			envMinLogLevel: "info",
-		},
-		{
-			name:       "Test log: Warning | Min level: Debug",
-			loggerType: "Warning",
-			args: args{
-				message: "This is a warning message",
-			},
-			wantLevel:      "warn",
-			wantMessage:    "This is a warning message",
-			stdErrExpect:   false,
-			stdOutExpect:   true,
-			envMinLogLevel: "debug",
-		},
-		{
-			name:       "Test log: Info",
-			loggerType: "Info",
-			args: args{
-				message: "This is a info message",
-			},
-			wantLevel:    "info",
-			wantMessage:  "This is a info message",
-			stdErrExpect: false,
-			stdOutExpect: true,
-		},
-		{
-			name:       "Test log: Info | Min level: Warn",
-			loggerType: "Info",
-			args: args{
-				message: "This is a info message",
-			},
-			wantLevel:      "",
-			wantMessage:    "",
-			stdErrExpect:   false,
-			stdOutExpect:   false,
-			envMinLogLevel: "warn",
-		},
-		{
-			name:       "Test log: Warning | Min level: Warn",
-			loggerType: "Warning",
-			args: args{
-				message: "This is a warning message",
-			},
-			wantLevel:      "warn",
-			wantMessage:    "This is a warning message",
-			stdErrExpect:   false,
-			stdOutExpect:   true,
-			envMinLogLevel: "warn",
-		},
-		{
-			name:       "Test log: Warning | Min level: Error",
-			loggerType: "Warning",
-			args: args{
-				message: "This is an error message",
-			},
-			wantLevel:      "",
-			wantMessage:    "",
-			stdErrExpect:   false,
-			stdOutExpect:   false,
-			envMinLogLevel: "error",
-		},
-		{
-			name:       "Test log: Debug | Min level: Debug",
-			loggerType: "Debug",
-			args: args{
-				message: "This is a debug message",
-			},
-			wantLevel:      "debug",
-			wantMessage:    "This is a debug message",
-			stdErrExpect:   false,
-			stdOutExpect:   true,
-			envMinLogLevel: "debug",
-		},
-	}
-
-	for _, tt := range tests {
-		suite.Run(tt.name, func() {
-			if tt.envMinLogLevel != "" {
-				os.Setenv("LOG_LEVEL", tt.envMinLogLevel)
-				defer os.Unsetenv("LOG_LEVEL")
-			}
-			testsLogger = NewLogger()
-
-			captureStdout := stdoutCapture{}
-			captureStdout.StartCapture()
-			captureStderr := stderrCapture{}
-			captureStderr.StartCapture()
-
-			reflect.ValueOf(testsLogger).MethodByName(tt.loggerType).Call([]reflect.Value{reflect.ValueOf(tt.args.message)})
-
-			stdoutOut, err := captureStdout.StopCapture()
-			if err != nil {
-				suite.T().Fatal(err)
-			}
-
-			stderrOut, err := captureStderr.StopCapture()
-			if err != nil {
-				suite.T().Fatal(err)
-			}
-
-			if tt.stdErrExpect && !tt.stdOutExpect {
-				gotLvl, gotMsg, err := getResponseValues(stderrOut, "short_message")
-				suite.NoError(err, "Failed in [STDERR]: "+tt.name)
-				suite.Equal(tt.wantLevel, gotLvl, "Failed in [STDERR]: "+tt.name)
-				suite.Equal(tt.wantMessage, gotMsg, "Failed in [STDERR]: "+tt.name)
-				suite.Equal("", stdoutOut, "Failed in [STDERR]: "+tt.name)
-			}
-			if tt.stdOutExpect && !tt.stdErrExpect {
-				gotLvl, gotMsg, err := getResponseValues(stdoutOut, "short_message")
-				suite.NoError(err, "Failed in [STDOUT]: "+tt.name)
-				suite.Equal(tt.wantLevel, gotLvl, "Failed in [STDOUT]: "+tt.name)
-				suite.Equal(tt.wantMessage, gotMsg, "Failed in [STDOUT]: "+tt.name)
-				suite.Equal("", stderrOut, "Failed in [STDOUT]: "+tt.name)
-			}
-			if !tt.stdErrExpect && !tt.stdOutExpect {
-				suite.Equal("", stderrOut, "Failed in [NEITHER]: "+tt.name)
-				suite.Equal("", stdoutOut, "Failed in [NEITHER]: "+tt.name)
-			}
-			os.Unsetenv("LOG_LEVEL")
-		})
-	}
-}
-
-func (suite *LoggingTestSuite) TestFullMessage() {
-	type args struct {
-		extraFields map[string]interface{}
-		message     string
-	}
-	extraFields := make(map[string]interface{})
-	extraFields["_full_message"] = "full message"
-
-	tests := []struct {
-		args           args
-		name           string
-		wantLevel      string
-		wantMessage    string
-		envMinLogLevel string
-		loggerType     string
-		stdOutExpect   bool
-		stdErrExpect   bool
-	}{
-		{
-			name:       "Test log: Error",
-			loggerType: "Error",
-			args: args{
-				message:     "This is a error message",
-				extraFields: extraFields,
-			},
-			wantLevel:    "error",
-			wantMessage:  extraFields["_full_message"].(string),
-			stdErrExpect: true,
-			stdOutExpect: false,
-		},
-		{
-			name:       "Test log: Info",
-			loggerType: "Info",
-			args: args{
-				message:     "This is a info message",
-				extraFields: extraFields,
-			},
-			wantMessage:  extraFields["_full_message"].(string),
-			stdErrExpect: false,
-			stdOutExpect: true,
-		},
-	}
-
-	for _, tt := range tests {
-		suite.Run(tt.name, func() {
-			if tt.envMinLogLevel != "" {
-				os.Setenv("LOG_LEVEL", tt.envMinLogLevel)
-				defer os.Unsetenv("LOG_LEVEL")
-			}
-			testsLogger = NewLogger()
-
-			captureStdout := stdoutCapture{}
-			captureStdout.StartCapture()
-			captureStderr := stderrCapture{}
-			captureStderr.StartCapture()
-
-			reflect.ValueOf(testsLogger).MethodByName(tt.loggerType).Call([]reflect.Value{
-				reflect.ValueOf(tt.args.message),
-				reflect.ValueOf(tt.args.extraFields),
-			})
-
-			stdoutOut, err := captureStdout.StopCapture()
-			if err != nil {
-				suite.T().Fatal(err)
-			}
-
-			stderrOut, err := captureStderr.StopCapture()
-			if err != nil {
-				suite.T().Fatal(err)
-			}
-
-			if tt.stdErrExpect && !tt.stdOutExpect {
-				_, gotMsg, err := getResponseValues(stderrOut, "_full_message")
-				suite.NoError(err, "Failed in [STDERR]: "+tt.name)
-				suite.Equal(tt.wantMessage, gotMsg, "Failed in [STDERR]: "+tt.name)
-			}
-			if tt.stdOutExpect && !tt.stdErrExpect {
-				_, gotMsg, err := getResponseValues(stdoutOut, "_full_message")
-				suite.NoError(err, "Failed in [STDOUT]: "+tt.name)
-				suite.Equal(tt.wantMessage, gotMsg, "Failed in [STDOUT]: "+tt.name)
-			}
-			os.Unsetenv("LOG_LEVEL")
-		})
-	}
-}
-
-func Test_getResponseValues(t *testing.T) {
-	type args struct {
-		sourceJson string
-	}
-	tests := []struct {
-		name       string
-		args       args
-		wantGotLvl string
-		wantGotMsg string
-		wantErr    bool
-	}{
-		{
-			name: "Test with json",
-			args: args{
-				sourceJson: `{"level": "debug", "short_message": "hello world"`,
-			},
-			wantGotLvl: "debug",
-			wantGotMsg: "hello world",
-			wantErr:    false,
-		},
-		{
-			name: "Test with json, wrong message field",
-			args: args{
-				sourceJson: `{"level": "debug", "message": "hello world"`,
-			},
-			wantGotLvl: "debug",
-			wantGotMsg: "",
-			wantErr:    true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			gotGotLvl, gotGotMsg, err := getResponseValues(tt.args.sourceJson, "short_message")
-			if (err != nil) != tt.wantErr {
-				t.Errorf("getResponseValues() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if gotGotLvl != tt.wantGotLvl {
-				t.Errorf("getResponseValues() gotGotLvl = %v, want %v", gotGotLvl, tt.wantGotLvl)
-			}
-			if gotGotMsg != tt.wantGotMsg {
-				t.Errorf("getResponseValues() gotGotMsg = %v, want %v", gotGotMsg, tt.wantGotMsg)
-			}
-		})
-	}
-}
-
-func getResponseValues(sourceJson string, key string) (gotLvl, gotMsg string, err error) {
-	gotLvl, err = jsonparser.GetString([]byte(sourceJson), "level")
-	if err != nil {
-		return
-	}
-	gotMsg, err = jsonparser.GetString([]byte(sourceJson), key)
-	return
-}

main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"os"
 	"strings"
+	"sync"
 
 	"github.com/gofiber/fiber/v2/middleware/proxy"
 	"github.com/gookit/goutil/envutil"
@@ -13,6 +14,7 @@ import (
 )
 
 var cfg *config
+var once sync.Once
 
 // function get value from the env where the value can be anything
 func getDetailsFromEnv[T any](key string, defaultValue T) T {
@@ -44,8 +46,14 @@ func parseConfig() {
 	c.Client.JWTRoleClaimPath = getDetailsFromEnv("JWT_ROLE_CLAIM_PATH", "")
 	c.Client.RoleFromHeader = getDetailsFromEnv("ROLE_FROM_HEADER", "")
 	c.Client.RoleRateLimit = getDetailsFromEnv("ROLE_RATE_LIMIT", false)
+	/* in-memory cache */
 	c.Cache.CacheEnable = getDetailsFromEnv("ENABLE_GLOBAL_CACHE", false)
 	c.Cache.CacheTTL = getDetailsFromEnv("CACHE_TTL", 60)
+	/* redis cache */
+	c.Cache.CacheRedisEnable = getDetailsFromEnv("ENABLE_REDIS_CACHE", false)
+	c.Cache.CacheRedisURL = getDetailsFromEnv("CACHE_REDIS_URL", "localhost:6379")
+	c.Cache.CacheRedisPassword = getDetailsFromEnv("CACHE_REDIS_PASSWORD", "")
+	c.Cache.CacheRedisDB = getDetailsFromEnv("CACHE_REDIS_DB", 0)
 	c.Security.BlockIntrospection = getDetailsFromEnv("BLOCK_SCHEMA_INTROSPECTION", false)
 	c.Security.IntrospectionAllowed = func() []string {
 		urls := getDetailsFromEnv("ALLOWED_INTROSPECTION", "")
@@ -75,11 +83,17 @@ func parseConfig() {
 	c.Api.BannedUsersFile = getDetailsFromEnv("BANNED_USERS_FILE", "/go/src/app/banned_users.json")
 	c.Server.PurgeOnCrawl = getDetailsFromEnv("PURGE_METRICS_ON_CRAWL", false)
 	c.Server.PurgeEvery = getDetailsFromEnv("PURGE_METRICS_ON_TIMER", 0)
+	c.HasuraEventCleaner.Enable = getDetailsFromEnv("HASURA_EVENT_CLEANER", false)
+	c.HasuraEventCleaner.ClearOlderThan = getDetailsFromEnv("HASURA_EVENT_CLEANER_OLDER_THAN", 1)
+	c.HasuraEventCleaner.EventMetadataDb = getDetailsFromEnv("HASURA_EVENT_METADATA_DB", "")
 	cfg = &c
 
 	enableCache() // takes close to no resources, but can be used with dynamic query cache
 	loadRatelimitConfig()
-	enableApi()
+	once.Do(func() {
+		go enableApi()
+		go enableHasuraEventCleaner()
+	})
 	prepareQueriesAndExemptions()
 }
 

main_test.go

@@ -3,9 +3,11 @@ package main
 import (
 	"os"
 	"testing"
+	"time"
 
 	"github.com/goccy/go-json"
 	"github.com/gofiber/fiber/v2"
+	libpack_cache "github.com/lukaszraczylo/graphql-monitoring-proxy/cache/memory"
 	libpack_logging "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
 	assertions "github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
@@ -32,7 +34,12 @@ func (suite *Tests) SetupTest() {
 			JSONDecoder:           json.Unmarshal,
 		},
 	)
+	cacheStats = &CacheStats{}
+
+	// Initialize a simple in-memory cache client for testing purposes
+	cfg.Cache.Client = libpack_cache.New(5 * time.Minute)
 	parseConfig()
+	enableApi()
 	StartMonitoringServer()
 	cfg.Logger = libpack_logging.NewLogger()
 	// Setup environment variables here if needed

monitoring/defaults.go

@@ -5,6 +5,9 @@ func (ms *MetricsSetup) RegisterDefaultMetrics() {
 	ms.RegisterMetricsCounter(MetricsFailed, nil)
 	ms.RegisterMetricsCounter(MetricsSkipped, nil)
 	ms.RegisterMetricsHistogram(MetricsDuration, nil)
+	ms.RegisterMetricsCounter(MetricsCacheHit, nil)
+	ms.RegisterMetricsCounter(MetricsCacheMiss, nil)
+	ms.RegisterMetricsCounter(MetricsQueriesCached, nil)
 }
 
 func (ms *MetricsSetup) RegisterGoMetrics() {

monitoring/helpers.go

@@ -5,55 +5,93 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"unicode"
 
 	libpack_config "github.com/lukaszraczylo/graphql-monitoring-proxy/config"
 )
 
 func (ms *MetricsSetup) get_metrics_name(name string, labels map[string]string) (complete_name string) {
-	var err error
-	if labels == nil {
-		labels = make(map[string]string)
+	const unknownPodName = "unknown"
+	var sb strings.Builder
+
+	// Prepare default labels without initializing a new map
+	podName := unknownPodName
+	if hn, err := os.Hostname(); err == nil {
+		podName = hn
 	}
-	labels["microservice"] = libpack_config.PKG_NAME
-	labels["pod"], err = os.Hostname()
-	if err != nil {
-		labels["pod"] = "unknown"
+	if labels == nil {
+		labels = map[string]string{
+			"microservice": libpack_config.PKG_NAME,
+			"pod":          podName,
+		}
+	} else {
+		if _, exists := labels["microservice"]; !exists {
+			labels["microservice"] = libpack_config.PKG_NAME
+		}
+		if _, exists := labels["pod"]; !exists {
+			labels["pod"] = podName
+		}
 	}
 
+	// Prefix handling
 	if ms.metrics_prefix != "" {
-		complete_name = ms.metrics_prefix + "_" + name
-	} else {
-		complete_name = name
+		sb.WriteString(ms.metrics_prefix)
+		sb.WriteString("_")
 	}
-	if labels != nil {
+	sb.WriteString(name)
+
+	// Append labels if any
+	if len(labels) > 0 {
+		sb.WriteString("{")
+
 		keys := make([]string, 0, len(labels))
 		for k := range labels {
 			keys = append(keys, k)
 		}
 		sort.Strings(keys)
-		complete_name += "{"
-		for _, k := range keys {
-			complete_name += k + "=\"" + labels[k] + "\","
+
+		for i, k := range keys {
+			if i > 0 {
+				sb.WriteString(",")
+			}
+			sb.WriteString(k)
+			sb.WriteString("=\"")
+			sb.WriteString(labels[k])
+			sb.WriteString("\"")
 		}
-		complete_name = strings.TrimSuffix(complete_name, ",")
-		complete_name += "}"
+		sb.WriteString("}")
 	}
-	return
+
+	return sb.String()
 }
 
 // validate_metrics_name validates the name of the metric to adhere to the Prometheus naming conventions
 // https://prometheus.io/docs/practices/naming/
 func validate_metrics_name(name string) error {
-	// replace all spaces with underscores and remove all other non-alphanumeric characters
-	name_new := strings.ReplaceAll(name, " ", "_")
-	name_new = strings.Map(func(r rune) rune {
-		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' {
-			return r
+	var sb strings.Builder // Use strings.Builder for efficient string concatenation
+
+	// Track if the last character was an underscore to avoid duplicate underscores
+	lastWasUnderscore := false
+
+	for _, r := range name {
+		// Convert spaces to underscores and skip non-alphanumeric characters except underscores
+		if r == ' ' || (unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_') {
+			if r == ' ' || r == '_' {
+				if lastWasUnderscore {
+					continue // Skip if the previous character was also an underscore
+				}
+				r = '_' // Convert spaces to underscores
+				lastWasUnderscore = true
+			} else {
+				lastWasUnderscore = false
+			}
+			sb.WriteRune(r) // Add valid characters to the builder
 		}
-		return -1
-	}, name_new)
-	name_new = strings.ReplaceAll(name_new, "__", "_")
-	name_new = strings.Trim(name_new, "_")
+	}
+	// Trim leading and trailing underscores
+	name_new := strings.Trim(sb.String(), "_")
+
+	// Check if the processed name matches the original input
 	if name_new != name {
 		return fmt.Errorf("Invalid metric name: %s, expected %s", name, name_new)
 	}
@@ -61,9 +99,31 @@ func validate_metrics_name(name string) error {
 }
 
 func compile_metrics_with_labels(name string, labels map[string]string) string {
-	metric_name := name
+	var totalLength int
+	totalLength += len(name)
 	for k, v := range labels {
-		metric_name += "_" + k + "_" + v
+		totalLength += len(k) + len(v) + 2
 	}
-	return metric_name
+
+	var sb strings.Builder
+	sb.Grow(totalLength + 1)
+
+	sb.WriteString(name)
+
+	// Collect keys and sort them
+	keys := make([]string, 0, len(labels))
+	for k := range labels {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// Append sorted key-value pairs to the builder
+	for _, k := range keys {
+		sb.WriteString("_")
+		sb.WriteString(k)
+		sb.WriteString("_")
+		sb.WriteString(labels[k])
+	}
+
+	return sb.String()
 }

monitoring/helpers_bench_test.go

@@ -0,0 +1,44 @@
+package libpack_monitoring
+
+import (
+	"testing"
+
+	libpack_config "github.com/lukaszraczylo/graphql-monitoring-proxy/config"
+)
+
+func BenchmarkGetMetricsName(b *testing.B) {
+	// Setup environment
+	libpack_config.PKG_NAME = "test_service"
+
+	ms := &MetricsSetup{metrics_prefix: "test_prefix"}
+
+	labels := map[string]string{
+		"env":    "production",
+		"region": "us-west-2",
+	}
+
+	// Run the benchmark
+	for n := 0; n < b.N; n++ {
+		ms.get_metrics_name("request_count", labels)
+	}
+}
+
+func BenchmarkCompileMetricsWithLabels(b *testing.B) {
+	labels := map[string]string{
+		"env":    "production",
+		"region": "us-west-2",
+		"app":    "api-server",
+	}
+
+	for n := 0; n < b.N; n++ {
+		compile_metrics_with_labels("request_count", labels)
+	}
+}
+
+func BenchmarkValidateMetricsName(b *testing.B) {
+	input := "valid metric name with special chars @#! and underscores__"
+
+	for n := 0; n < b.N; n++ {
+		validate_metrics_name(input)
+	}
+}

monitoring/helpers_test.go

@@ -0,0 +1,143 @@
+package libpack_monitoring
+
+import (
+	"os"
+	"testing"
+
+	libpack_config "github.com/lukaszraczylo/graphql-monitoring-proxy/config"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetMetricsName(t *testing.T) {
+	ms := &MetricsSetup{metrics_prefix: "prefix"}
+	libpack_config.PKG_NAME = "example_microservice"
+
+	tests := []struct {
+		name           string
+		metricName     string
+		labels         map[string]string
+		expectedOutput string
+	}{
+		{
+			name:           "No labels",
+			metricName:     "test_metric",
+			labels:         nil,
+			expectedOutput: "prefix_test_metric{microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:       "With labels",
+			metricName: "test_metric",
+			labels: map[string]string{
+				"label1": "value1",
+				"label2": "value2",
+			},
+			expectedOutput: "prefix_test_metric{label1=\"value1\",label2=\"value2\",microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:       "Alphabetical order labels",
+			metricName: "test_metric",
+			labels: map[string]string{
+				"label2": "value2",
+				"label1": "value1",
+			},
+			expectedOutput: "prefix_test_metric{label1=\"value1\",label2=\"value2\",microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:           "Empty metric name",
+			metricName:     "",
+			labels:         nil,
+			expectedOutput: "prefix_{microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:           "Empty labels map",
+			metricName:     "test_metric",
+			labels:         map[string]string{},
+			expectedOutput: "prefix_test_metric{microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:       "Single label",
+			metricName: "test_metric",
+			labels: map[string]string{
+				"label1": "value1",
+			},
+			expectedOutput: "prefix_test_metric{label1=\"value1\",microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:       "Multiple labels with special characters",
+			metricName: "test_metric",
+			labels: map[string]string{
+				"label-2": "value-2",
+				"label_1": "value_1",
+			},
+			expectedOutput: "prefix_test_metric{label-2=\"value-2\",label_1=\"value_1\",microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+		{
+			name:       "Prefix only",
+			metricName: "",
+			labels: map[string]string{
+				"label1": "value1",
+			},
+			expectedOutput: "prefix_{label1=\"value1\",microservice=\"example_microservice\",pod=\"" + getPodName() + "\"}",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := ms.get_metrics_name(tt.metricName, tt.labels)
+			assert.Equal(t, tt.expectedOutput, result)
+		})
+	}
+}
+
+func TestCompileMetricsWithLabels(t *testing.T) {
+	tests := []struct {
+		name   string
+		labels map[string]string
+		want   string
+	}{
+		{"request_count", map[string]string{"env": "production", "region": "us-west-2"}, "request_count_env_production_region_us-west-2"},
+		{"metric_name", map[string]string{}, "metric_name"},
+		{"metric_name", nil, "metric_name"},
+		{"metric_name", map[string]string{"key1": "value1"}, "metric_name_key1_value1"},
+		{"metric_name", map[string]string{"k": "v", "key2": "value2"}, "metric_name_k_v_key2_value2"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := compile_metrics_with_labels(tt.name, tt.labels); got != tt.want {
+				t.Errorf("compile_metrics_with_labels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestValidateMetricsName(t *testing.T) {
+	tests := []struct {
+		name    string
+		input   string
+		wantErr bool
+	}{
+		{"Valid name", "valid_metric_name", false},
+		{"Name with spaces", "valid metric name", true},
+		{"Name with special chars", "valid@metric#name!", true},
+		{"Name with leading underscore", "_valid_metric_name", true},
+		{"Name with trailing underscore", "valid_metric_name_", true},
+		{"Name with consecutive underscores", "valid__metric__name", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := validate_metrics_name(tt.input); (err != nil) != tt.wantErr {
+				t.Errorf("validate_metrics_name() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func getPodName() string {
+	podName, err := os.Hostname()
+	if err != nil {
+		return "unknown"
+	}
+	return podName
+}

monitoring/structs.go

@@ -7,4 +7,8 @@ const (
 	MetricsSkipped       = "requests_skipped"
 	MetricsExecutedQuery = "executed_query"
 	MetricsTimedQuery    = "timed_query"
+
+	MetricsCacheHit      = "cache_hit"
+	MetricsCacheMiss     = "cache_miss"
+	MetricsQueriesCached = "cached_queries"
 )

ratelimit.go

@@ -27,7 +27,7 @@ var ratelimit_intervals = map[string]time.Duration{
 }
 
 func loadRatelimitConfig() error {
-	paths := []string{"/app/ratelimit.json", "./ratelimit.json", "./static/default-ratelimit.json"}
+	paths := []string{"/go/src/app/ratelimit.json", "./ratelimit.json", "./static/app/default-ratelimit.json"}
 
 	for _, path := range paths {
 		err := loadConfigFromPath(path)
@@ -37,7 +37,7 @@ func loadRatelimitConfig() error {
 		cfg.Logger.Debug("Failed to load config", map[string]interface{}{"path": path, "error": err})
 	}
 
-	cfg.Logger.Error("Rate limit config not found")
+	cfg.Logger.Error("Rate limit config not found", map[string]interface{}{"paths": paths})
 	return os.ErrNotExist
 }
 

server.go

@@ -16,6 +16,7 @@ import (
 
 // StartHTTPProxy starts the HTTP and points it to the GraphQL server.
 func StartHTTPProxy() {
+	cfg.Logger.Debug("Starting the HTTP proxy", nil)
 	server := fiber.New(fiber.Config{
 		DisableStartupMessage: true,
 		AppName:               fmt.Sprintf("GraphQL Monitoring Proxy - %s v%s", libpack_config.PKG_NAME, libpack_config.PKG_VERSION),
@@ -75,7 +76,7 @@ func healthCheck(c *fiber.Ctx) error {
 			return err
 		}
 	}
-	cfg.Logger.Debug("Health check returning OK")
+	cfg.Logger.Debug("Health check returning OK", nil)
 	c.Status(200).SendString("Health check OK")
 	return nil
 }
@@ -121,10 +122,12 @@ func processGraphQLRequest(c *fiber.Ctx) error {
 	}
 
 	if parsedResult.shouldIgnore {
-		cfg.Logger.Debug("Request passed as-is - probably not a GraphQL")
+		cfg.Logger.Debug("Request passed as-is - probably not a GraphQL", nil)
 		return proxyTheRequest(c, parsedResult.activeEndpoint)
 	}
 
+	calculatedQueryHash := calculateHash(c)
+
 	if parsedResult.cacheTime > 0 {
 		cfg.Logger.Debug("Cache time set via query", map[string]interface{}{"cacheTime": parsedResult.cacheTime})
 	} else {
@@ -142,25 +145,38 @@ func processGraphQLRequest(c *fiber.Ctx) error {
 
 	if parsedResult.cacheRefresh {
 		cfg.Logger.Debug("Cache refresh requested via query", map[string]interface{}{"user_id": extractedUserID, "request_uuid": c.Locals("request_uuid")})
-		cacheDelete(calculateHash(c))
+		cacheDelete(calculatedQueryHash)
 	}
 
 	// Handling Cache Logic
-	if parsedResult.cacheRequest || cfg.Cache.CacheEnable {
+	if parsedResult.cacheRequest || cfg.Cache.CacheEnable || cfg.Cache.CacheRedisEnable {
 		cfg.Logger.Debug("Cache enabled", map[string]interface{}{"via_query": parsedResult.cacheRequest, "via_env": cfg.Cache.CacheEnable})
-		queryCacheHash = calculateHash(c)
+		queryCacheHash = calculatedQueryHash
 
 		if cachedResponse := cacheLookup(queryCacheHash); cachedResponse != nil {
+			cfg.Monitoring.Increment(libpack_monitoring.MetricsCacheHit, nil)
 			cfg.Logger.Debug("Cache hit", map[string]interface{}{"hash": queryCacheHash, "user_id": extractedUserID, "request_uuid": c.Locals("request_uuid")})
 			c.Request().Header.Add("X-Cache-Hit", "true")
-			c.Send(cachedResponse)
+			err := c.Send(cachedResponse)
+			if err != nil {
+				cfg.Logger.Error("Can't send the cached response", map[string]interface{}{"error": err.Error()})
+				cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
+				c.Status(500).SendString("Can't send the cached response - try again later")
+			}
 			wasCached = true
 		} else {
+			cfg.Monitoring.Increment(libpack_monitoring.MetricsCacheMiss, nil)
 			cfg.Logger.Debug("Cache miss", map[string]interface{}{"hash": queryCacheHash, "user_id": extractedUserID, "request_uuid": c.Locals("request_uuid")})
 			proxyAndCacheTheRequest(c, queryCacheHash, parsedResult.cacheTime, parsedResult.activeEndpoint)
 		}
 	} else {
-		proxyTheRequest(c, parsedResult.activeEndpoint)
+		err := proxyTheRequest(c, parsedResult.activeEndpoint)
+		if err != nil {
+			cfg.Logger.Error("Can't proxy the request", map[string]interface{}{"error": err.Error()})
+			cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
+			c.Status(500).SendString("Can't proxy the request - try again later")
+			return nil
+		}
 	}
 
 	timeTaken := time.Since(startTime)
@@ -180,7 +196,8 @@ func proxyAndCacheTheRequest(c *fiber.Ctx, queryCacheHash string, cacheTime int,
 		c.Status(500).SendString("Can't proxy the request - try again later")
 		return
 	}
-	cfg.Cache.CacheClient.Set(queryCacheHash, c.Response().Body(), time.Duration(cacheTime)*time.Second)
+	cacheStoreWithTTL(queryCacheHash, c.Response().Body(), time.Duration(cacheTime)*time.Second)
+	cfg.Monitoring.Increment(libpack_monitoring.MetricsQueriesCached, nil)
 	c.Send(c.Response().Body())
 }
 

static/app/banned_users.json

@@ -0,0 +1 @@
+{}

static/kubernetes-single-deployment-with-ro.yaml

@@ -80,6 +80,9 @@ spec:
           env:
             - name: HASURA_GRAPHQL_DATABASE_URL
               value: postgres://postgres:xxx@yyy.read-only:5432/postgres
+              # POINT METADATA TO THE RW database (!!!)
+            - name: HASURA_GRAPHQL_METADATA_DATABASE_URL
+              value: postgres://postgres:xxx@yyy:5432/postgres
             - name: HASURA_GRAPHQL_ENABLE_CONSOLE
               value: "true"
             - name: HASURA_GRAPHQL_DEV_MODE
@@ -92,7 +95,7 @@ spec:
               value: "20"
             - name: HASURA_GRAPHQL_LOG_LEVEL
               value: "error"
-            - name: HASURA_PORT
+            - name: HASURA_GRAPHQL_SERVER_PORT
               value: "8088"
 
         - name: graphql-proxy

struct_config.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	graphql "github.com/lukaszraczylo/go-simple-graphql"
-	libpack_cache "github.com/lukaszraczylo/graphql-monitoring-proxy/cache"
 	libpack_logging "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
 	libpack_monitoring "github.com/lukaszraczylo/graphql-monitoring-proxy/monitoring"
 	"github.com/valyala/fasthttp"
@@ -10,11 +9,6 @@ import (
 
 // config is a struct that holds the configuration of the application.
 type config struct {
-	Cache struct {
-		CacheClient *libpack_cache.Cache
-		CacheTTL    int
-		CacheEnable bool
-	}
 	Logger     *libpack_logging.LogConfig
 	Monitoring *libpack_monitoring.MetricsSetup
 	Api        struct{ BannedUsersFile string }
@@ -32,6 +26,20 @@ type config struct {
 		IntrospectionAllowed []string
 		BlockIntrospection   bool
 	}
+	HasuraEventCleaner struct {
+		EventMetadataDb string
+		ClearOlderThan  int
+		Enable          bool
+	}
+	Cache struct {
+		Client             CacheClient
+		CacheRedisURL      string
+		CacheRedisPassword string
+		CacheTTL           int
+		CacheRedisDB       int
+		CacheEnable        bool
+		CacheRedisEnable   bool
+	}
 	Server struct {
 		HostGraphQL         string
 		HostGraphQLReadOnly string