From b314a255494ed81d78ec0a222be72e36b4b4bf68 Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Mon, 3 Nov 2025 21:02:50 +0000 Subject: [PATCH] Fix retry patterns --- README.md | 41 +++++++++++++++++++- execute_query.go | 97 ++++++++++++++++++++++++++++++++++++++---------- main.go | 36 +++++++++++++----- structs.go | 23 ++++++------ 4 files changed, 156 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3bffc74..7f692a9 100644 --- a/README.md +++ b/README.md @@ -37,15 +37,52 @@ Therefore, I present you the simple client to which you can copy & paste your gr * `GRAPHQL_CACHE_TTL` - Cache TTL in seconds for SELECT type of queries. Default: `5` * `GRAPHQL_OUTPUT` - Output format. Default: `string`, available: `byte`, `string`, `mapstring` * `LOG_LEVEL` - Logging level. Default: `info` available: `debug`, `info`, `warn`, `error` -* `GRAPHQL_RETRIES_ENABLE` - Should retries be enabled? Default: `true` -* `GRAPHQL_RETRIES_NUMBER` - Number of retries: Default: `1` +* `GRAPHQL_RETRIES_ENABLE` - Should retries be enabled? Default: `false` +* `GRAPHQL_RETRIES_NUMBER` - Number of retries: Default: `3` * `GRAPHQL_RETRIES_DELAY` - Delay in retries in milliseconds. Default: `250` +* `GRAPHQL_RETRIES_PATTERNS` - Comma-separated error patterns that trigger retries. Default: `postgres,connection,timeout,transaction,could not,temporarily unavailable,deadlock` ### Modifiers on the fly * `gql.SetEndpoint('your-endpoint-url')` - modifies endpoint, without the need to set the environment variable * `gql.SetOutput('byte')` - modifies output format, without the need to set the environment variable +### Retries + +The library supports automatic retries for both HTTP-level errors and GraphQL errors that match configurable patterns. + +**Retry behavior:** +- **HTTP errors** (status codes outside 200-299): Always retried if retries are enabled +- **GraphQL errors**: Only retried if the error message contains one of the configured patterns +- **Non-matching errors**: Fail immediately without retrying (e.g., validation errors, field not found) + +**Configuring retryable patterns:** + +By default, errors containing these patterns will trigger retries: +- `postgres` - Database connection/transaction errors +- `connection` - Network connection issues +- `timeout` - Request timeout errors +- `transaction` - Database transaction conflicts +- `could not` - Generic transient failures +- `temporarily unavailable` - Service unavailability +- `deadlock` - Database deadlocks + +You can customize these patterns using the `GRAPHQL_RETRIES_PATTERNS` environment variable: + +```bash +export GRAPHQL_RETRIES_PATTERNS="postgres,connection,timeout,custom-error-pattern" +``` + +**Example:** To handle flaky Hasura database connections, enable retries: + +```bash +export GRAPHQL_RETRIES_ENABLE=true +export GRAPHQL_RETRIES_NUMBER=3 +export GRAPHQL_RETRIES_DELAY=250 +``` + +This will retry GraphQL queries up to 3 times when postgres transaction errors occur, with exponential backoff starting at 250ms. + ### Cache You have two options to enable the cache: diff --git a/execute_query.go b/execute_query.go index 5078e09..9f3c050 100644 --- a/execute_query.go +++ b/execute_query.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "strings" "time" "unicode/utf8" @@ -32,6 +33,47 @@ func max(a, b int) int { return b } +// isRetryableError checks if GraphQL errors contain patterns that indicate a transient failure +// that should be retried (e.g., database connection issues, timeouts, transaction conflicts) +func isRetryableError(errors []struct { + Message interface{} `json:"message"` +}, patterns []string) bool { + if len(errors) == 0 || len(patterns) == 0 { + return false + } + + for _, err := range errors { + // Try to extract error message as string + var msgStr string + switch msg := err.Message.(type) { + case string: + msgStr = msg + case map[string]interface{}: + // Handle nested error structures (e.g., {"message": "...", "extensions": {...}}) + if nestedMsg, ok := msg["message"].(string); ok { + msgStr = nestedMsg + } + default: + // If we can't convert to string, try fmt.Sprint as fallback + msgStr = fmt.Sprint(msg) + } + + if msgStr == "" { + continue + } + + // Check if error message contains any of the configured retry patterns (case-insensitive) + msgLower := strings.ToLower(msgStr) + for _, pattern := range patterns { + if strings.Contains(msgLower, strings.ToLower(pattern)) { + return true + } + } + } + + return false +} + var ( // Shared HTTP transport with optimized settings defaultTransport = &http.Transport{ @@ -331,6 +373,38 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) { return fmt.Errorf("error unmarshalling HTTP response: %w", err) } + // Check for GraphQL errors in the response + if len(queryResult.Errors) > 0 { + // Determine if this is a retryable error based on configured patterns + if isRetryableError(queryResult.Errors, qe.retries_patterns) { + // Log as warning since we'll retry + qe.Logger.Warning(&libpack_logger.LogMessage{ + Message: "Retryable GraphQL error detected", + Pairs: map[string]interface{}{"errors": queryResult.Errors}, + }) + // Return error to trigger retry mechanism + return fmt.Errorf("retryable error executing query: %s", queryResult.Errors) + } + + // Non-retryable error - log and fail immediately without retrying + qe.Logger.Error(&libpack_logger.LogMessage{ + Message: "Non-retryable GraphQL error", + Pairs: map[string]interface{}{"errors": queryResult.Errors}, + }) + // Use retry.Unrecoverable to skip remaining retry attempts + return retry.Unrecoverable(fmt.Errorf("error executing query: %s", queryResult.Errors)) + } + + // Check for null data in the response + if queryResult.Data == nil { + qe.Logger.Error(&libpack_logger.LogMessage{ + Message: "GraphQL query returned no data", + Pairs: map[string]interface{}{"error": "data field is null"}, + }) + // Null data is typically not retryable (indicates query structure issue) + return retry.Unrecoverable(errors.New("error executing query: no data")) + } + return nil }, retry.OnRetry(func(n uint, err error) { @@ -346,33 +420,18 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) { retry.LastErrorOnly(true), ) if err != nil { - qe.Logger.Debug(&libpack_logger.LogMessage{ - Message: "Error executing HTTP request", + qe.Logger.Error(&libpack_logger.LogMessage{ + Message: "Query execution failed after retries", Pairs: map[string]interface{}{"error": err.Error()}, }) return nil, err } - if len(queryResult.Errors) > 0 { - qe.Logger.Debug(&libpack_logger.LogMessage{ - Message: "Error executing query", - Pairs: map[string]interface{}{"error": queryResult.Errors}, - }) - return nil, fmt.Errorf("error executing query: %s", queryResult.Errors) - } - - if queryResult.Data == nil { - qe.Logger.Debug(&libpack_logger.LogMessage{ - Message: "Error executing query", - Pairs: map[string]interface{}{"error": "no data"}, - }) - return nil, errors.New("error executing query: no data") - } - + // At this point, we have successfully executed the query and validated the response // Marshal queryResult.Data jsonData, err := json.Marshal(queryResult.Data) if err != nil { - qe.Logger.Debug(&libpack_logger.LogMessage{ + qe.Logger.Error(&libpack_logger.LogMessage{ Message: "Error marshalling query result", Pairs: map[string]interface{}{"error": err.Error(), "data": queryResult.Data}, }) diff --git a/main.go b/main.go index aad8dd3..14c465f 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package gql import ( "net/http" + "strings" "time" "github.com/gookit/goutil/envutil" @@ -9,6 +10,22 @@ import ( logging "github.com/lukaszraczylo/go-simple-graphql/logging" ) +// parseRetryPatterns parses a comma-separated string of retry patterns into a slice +func parseRetryPatterns(patterns string) []string { + if patterns == "" { + return []string{} + } + parts := strings.Split(patterns, ",") + result := make([]string, 0, len(parts)) + for _, p := range parts { + trimmed := strings.TrimSpace(p) + if trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + func NewConnection() (b *BaseClient) { // Read LOG_LEVEL environment variable logLevelStr := envutil.Getenv("LOG_LEVEL", "info") @@ -30,15 +47,16 @@ func NewConnection() (b *BaseClient) { }) b = &BaseClient{ - endpoint: envutil.Getenv("GRAPHQL_ENDPOINT", "https://api.github.com/graphql"), - responseType: envutil.Getenv("GRAPHQL_OUTPUT", "string"), - Logger: logger, - cache: cache.New(time.Duration(envutil.GetInt("GRAPHQL_CACHE_TTL", 5)) * time.Second), - cache_global: envutil.GetBool("GRAPHQL_CACHE_ENABLED", false), - retries_enable: envutil.GetBool("GRAPHQL_RETRIES_ENABLE", false), - retries_delay: time.Duration(envutil.GetInt("GRAPHQL_RETRIES_DELAY", 250) * int(time.Millisecond)), - retries_number: envutil.GetInt("GRAPHQL_RETRIES_NUMBER", 3), - minify_queries: envutil.GetBool("GRAPHQL_MINIFY_QUERIES", true), // Default: enabled for production efficiency + endpoint: envutil.Getenv("GRAPHQL_ENDPOINT", "https://api.github.com/graphql"), + responseType: envutil.Getenv("GRAPHQL_OUTPUT", "string"), + Logger: logger, + cache: cache.New(time.Duration(envutil.GetInt("GRAPHQL_CACHE_TTL", 5)) * time.Second), + cache_global: envutil.GetBool("GRAPHQL_CACHE_ENABLED", false), + retries_enable: envutil.GetBool("GRAPHQL_RETRIES_ENABLE", false), + retries_delay: time.Duration(envutil.GetInt("GRAPHQL_RETRIES_DELAY", 250) * int(time.Millisecond)), + retries_number: envutil.GetInt("GRAPHQL_RETRIES_NUMBER", 3), + retries_patterns: parseRetryPatterns(envutil.Getenv("GRAPHQL_RETRIES_PATTERNS", "postgres,connection,timeout,transaction,could not,temporarily unavailable,deadlock")), + minify_queries: envutil.GetBool("GRAPHQL_MINIFY_QUERIES", true), // Default: enabled for production efficiency } b.client = b.createHttpClient() b.Logger.Debug(&logging.LogMessage{ diff --git a/structs.go b/structs.go index 7a88dc7..b798626 100644 --- a/structs.go +++ b/structs.go @@ -9,17 +9,18 @@ import ( ) type BaseClient struct { - cache *cache.Cache - Logger *logging.Logger - client *http.Client - endpoint string - responseType string - retries_delay time.Duration - retries_number int - MaxGoRoutines int - cache_global bool - retries_enable bool - minify_queries bool // Enable GraphQL query minification (default: true) + cache *cache.Cache + Logger *logging.Logger + client *http.Client + endpoint string + responseType string + retries_delay time.Duration + retries_number int + retries_patterns []string // Error patterns that should trigger retries + MaxGoRoutines int + cache_global bool + retries_enable bool + minify_queries bool // Enable GraphQL query minification (default: true) } type Query struct {