Compare commits

..
1 Commits
Author SHA1 Message Date
lukaszraczylo b314a25549 Fix retry patterns 2025-11-03 21:02:50 +00:00
12 changed files with 209 additions and 425 deletions
+39 -2
View File
@@ -37,15 +37,52 @@ Therefore, I present you the simple client to which you can copy & paste your gr
* `GRAPHQL_CACHE_TTL` - Cache TTL in seconds for SELECT type of queries. Default: `5`
* `GRAPHQL_OUTPUT` - Output format. Default: `string`, available: `byte`, `string`, `mapstring`
* `LOG_LEVEL` - Logging level. Default: `info` available: `debug`, `info`, `warn`, `error`
* `GRAPHQL_RETRIES_ENABLE` - Should retries be enabled? Default: `true`
* `GRAPHQL_RETRIES_NUMBER` - Number of retries: Default: `1`
* `GRAPHQL_RETRIES_ENABLE` - Should retries be enabled? Default: `false`
* `GRAPHQL_RETRIES_NUMBER` - Number of retries: Default: `3`
* `GRAPHQL_RETRIES_DELAY` - Delay in retries in milliseconds. Default: `250`
* `GRAPHQL_RETRIES_PATTERNS` - Comma-separated error patterns that trigger retries. Default: `postgres,connection,timeout,transaction,could not,temporarily unavailable,deadlock`
### Modifiers on the fly
* `gql.SetEndpoint('your-endpoint-url')` - modifies endpoint, without the need to set the environment variable
* `gql.SetOutput('byte')` - modifies output format, without the need to set the environment variable
### Retries
The library supports automatic retries for both HTTP-level errors and GraphQL errors that match configurable patterns.
**Retry behavior:**
- **HTTP errors** (status codes outside 200-299): Always retried if retries are enabled
- **GraphQL errors**: Only retried if the error message contains one of the configured patterns
- **Non-matching errors**: Fail immediately without retrying (e.g., validation errors, field not found)
**Configuring retryable patterns:**
By default, errors containing these patterns will trigger retries:
- `postgres` - Database connection/transaction errors
- `connection` - Network connection issues
- `timeout` - Request timeout errors
- `transaction` - Database transaction conflicts
- `could not` - Generic transient failures
- `temporarily unavailable` - Service unavailability
- `deadlock` - Database deadlocks
You can customize these patterns using the `GRAPHQL_RETRIES_PATTERNS` environment variable:
```bash
export GRAPHQL_RETRIES_PATTERNS="postgres,connection,timeout,custom-error-pattern"
```
**Example:** To handle flaky Hasura database connections, enable retries:
```bash
export GRAPHQL_RETRIES_ENABLE=true
export GRAPHQL_RETRIES_NUMBER=3
export GRAPHQL_RETRIES_DELAY=250
```
This will retry GraphQL queries up to 3 times when postgres transaction errors occur, with exponential backoff starting at 250ms.
### Cache
You have two options to enable the cache:
+96 -82
View File
@@ -8,7 +8,7 @@ import (
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"unicode/utf8"
@@ -17,8 +17,6 @@ import (
libpack_logger "github.com/lukaszraczylo/go-simple-graphql/logging"
)
// Size limit constants are defined in query.go
// min returns the smaller of two integers
func min(a, b int) int {
if a < b {
@@ -35,38 +33,64 @@ func max(a, b int) int {
return b
}
// getDefaultClient creates a default HTTP client with optimized settings
// This is called dynamically to ensure environment variables are read at runtime
func getDefaultClient() *http.Client {
return &http.Client{
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 90 * time.Second,
DisableCompression: true, // Disable automatic request compression to prevent "trailing garbage" errors
ForceAttemptHTTP2: true,
TLSClientConfig: getTLSConfig(),
},
Timeout: 30 * time.Second,
// isRetryableError checks if GraphQL errors contain patterns that indicate a transient failure
// that should be retried (e.g., database connection issues, timeouts, transaction conflicts)
func isRetryableError(errors []struct {
Message interface{} `json:"message"`
}, patterns []string) bool {
if len(errors) == 0 || len(patterns) == 0 {
return false
}
for _, err := range errors {
// Try to extract error message as string
var msgStr string
switch msg := err.Message.(type) {
case string:
msgStr = msg
case map[string]interface{}:
// Handle nested error structures (e.g., {"message": "...", "extensions": {...}})
if nestedMsg, ok := msg["message"].(string); ok {
msgStr = nestedMsg
}
default:
// If we can't convert to string, try fmt.Sprint as fallback
msgStr = fmt.Sprint(msg)
}
if msgStr == "" {
continue
}
// Check if error message contains any of the configured retry patterns (case-insensitive)
msgLower := strings.ToLower(msgStr)
for _, pattern := range patterns {
if strings.Contains(msgLower, strings.ToLower(pattern)) {
return true
}
}
}
return false
}
// getTLSConfig returns TLS configuration based on environment settings
// By default, TLS verification is enabled for security.
// Set GRAPHQL_INSECURE_SKIP_VERIFY=true to disable verification (NOT RECOMMENDED for production)
func getTLSConfig() *tls.Config {
skipVerify := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY") == "true"
if skipVerify {
// Log warning when running in insecure mode
fmt.Fprintf(os.Stderr, "WARNING: TLS certificate verification is disabled. This is insecure and should not be used in production.\n")
var (
// Shared HTTP transport with optimized settings
defaultTransport = &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 90 * time.Second,
DisableCompression: true, // Disable automatic request compression to prevent "trailing garbage" errors
ForceAttemptHTTP2: true,
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // Skip TLS verification for test environments
}
return &tls.Config{
InsecureSkipVerify: skipVerify,
MinVersion: tls.VersionTLS12, // Enforce minimum TLS 1.2 for security
// Shared HTTP client with timeouts
defaultClient = &http.Client{
Transport: defaultTransport,
Timeout: 30 * time.Second,
}
}
)
func (qe *QueryExecutor) executeQuery() ([]byte, error) {
// Reuse buffer from pool to avoid allocations
@@ -76,10 +100,7 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
buf.Write(qe.Query)
qe.mu.RLock()
endpoint := qe.endpoint
qe.mu.RUnlock()
httpRequest, err := http.NewRequest(http.MethodPost, endpoint, nil)
httpRequest, err := http.NewRequest(http.MethodPost, qe.endpoint, nil)
if err != nil {
qe.Logger.Error(&libpack_logger.LogMessage{
Message: "Can't create HTTP request",
@@ -103,9 +124,7 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
retriesMax := 1
if qe.Retries {
qe.mu.RLock()
retriesMax = qe.retries_number
qe.mu.RUnlock()
}
var queryResult queryResults
@@ -123,8 +142,8 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
"body_size": len(requestBody),
"contains_jsonQuery": bytes.Contains(requestBody, []byte(`"jsonQuery"`)),
"contains_base64": bytes.Contains(requestBody, []byte("eyJ")),
"first_100_chars": sanitizeForLogging(string(requestBody[:min(100, len(requestBody))])),
"last_50_chars": sanitizeForLogging(string(requestBody[max(0, len(requestBody)-50):])),
"first_100_chars": string(requestBody[:min(100, len(requestBody))]),
"last_50_chars": string(requestBody[max(0, len(requestBody)-50):]),
"is_valid_utf8": utf8.Valid(requestBody),
"has_null_bytes": bytes.Contains(requestBody, []byte{0}),
"has_control_chars": hasControlChars(requestBody),
@@ -155,11 +174,9 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
})
// Use default client if custom client is not set
qe.mu.RLock()
client := qe.client
qe.mu.RUnlock()
if client == nil {
client = getDefaultClient()
client = defaultClient
}
httpResponse, err := client.Do(httpRequest)
if err != nil {
@@ -184,32 +201,16 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
},
})
// Check Content-Length header if available for early size validation
if contentLength := httpResponse.ContentLength; contentLength > 0 && contentLength > MaxResponseSize {
return fmt.Errorf("response size %d exceeds maximum allowed size of %d bytes", contentLength, MaxResponseSize)
}
// First, read the entire response body into a buffer with size limit
// First, read the entire response body into a buffer
rawBuf := bufferPool.Get().(*bytes.Buffer)
rawBuf.Reset()
defer bufferPool.Put(rawBuf)
// Use LimitedReader to prevent reading more than MaxResponseSize
limitedReader := &io.LimitedReader{
R: httpResponse.Body,
N: MaxResponseSize + 1, // Read one extra byte to detect oversized responses
}
n, err := io.Copy(rawBuf, limitedReader)
_, err = io.Copy(rawBuf, httpResponse.Body)
if err != nil {
return fmt.Errorf("error reading HTTP response body: %w", err)
}
// Check if response exceeded size limit
if n > MaxResponseSize {
return fmt.Errorf("response size exceeds maximum allowed size of %d bytes", MaxResponseSize)
}
rawData := rawBuf.Bytes()
qe.Logger.Debug(&libpack_logger.LogMessage{
Message: "Read raw response data",
@@ -372,6 +373,38 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
return fmt.Errorf("error unmarshalling HTTP response: %w", err)
}
// Check for GraphQL errors in the response
if len(queryResult.Errors) > 0 {
// Determine if this is a retryable error based on configured patterns
if isRetryableError(queryResult.Errors, qe.retries_patterns) {
// Log as warning since we'll retry
qe.Logger.Warning(&libpack_logger.LogMessage{
Message: "Retryable GraphQL error detected",
Pairs: map[string]interface{}{"errors": queryResult.Errors},
})
// Return error to trigger retry mechanism
return fmt.Errorf("retryable error executing query: %s", queryResult.Errors)
}
// Non-retryable error - log and fail immediately without retrying
qe.Logger.Error(&libpack_logger.LogMessage{
Message: "Non-retryable GraphQL error",
Pairs: map[string]interface{}{"errors": queryResult.Errors},
})
// Use retry.Unrecoverable to skip remaining retry attempts
return retry.Unrecoverable(fmt.Errorf("error executing query: %s", queryResult.Errors))
}
// Check for null data in the response
if queryResult.Data == nil {
qe.Logger.Error(&libpack_logger.LogMessage{
Message: "GraphQL query returned no data",
Pairs: map[string]interface{}{"error": "data field is null"},
})
// Null data is typically not retryable (indicates query structure issue)
return retry.Unrecoverable(errors.New("error executing query: no data"))
}
return nil
},
retry.OnRetry(func(n uint, err error) {
@@ -382,42 +415,23 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) {
}),
retry.Attempts(uint(retriesMax)),
retry.DelayType(retry.BackOffDelay),
retry.Delay(func() time.Duration {
qe.mu.RLock()
defer qe.mu.RUnlock()
return qe.retries_delay
}()),
retry.Delay(time.Duration(qe.retries_delay)),
retry.MaxDelay(10*time.Second),
retry.LastErrorOnly(true),
)
if err != nil {
qe.Logger.Debug(&libpack_logger.LogMessage{
Message: "Error executing HTTP request",
qe.Logger.Error(&libpack_logger.LogMessage{
Message: "Query execution failed after retries",
Pairs: map[string]interface{}{"error": err.Error()},
})
return nil, err
}
if len(queryResult.Errors) > 0 {
qe.Logger.Debug(&libpack_logger.LogMessage{
Message: "Error executing query",
Pairs: map[string]interface{}{"error": queryResult.Errors},
})
return nil, fmt.Errorf("error executing query: %s", queryResult.Errors)
}
if queryResult.Data == nil {
qe.Logger.Debug(&libpack_logger.LogMessage{
Message: "Error executing query",
Pairs: map[string]interface{}{"error": "no data"},
})
return nil, errors.New("error executing query: no data")
}
// At this point, we have successfully executed the query and validated the response
// Marshal queryResult.Data
jsonData, err := json.Marshal(queryResult.Data)
if err != nil {
qe.Logger.Debug(&libpack_logger.LogMessage{
qe.Logger.Error(&libpack_logger.LogMessage{
Message: "Error marshalling query result",
Pairs: map[string]interface{}{"error": err.Error(), "data": queryResult.Data},
})
+8 -37
View File
@@ -2,9 +2,7 @@ package gql
import (
"crypto/tls"
"fmt"
"net/http"
"os"
"strings"
"time"
@@ -12,7 +10,7 @@ import (
"golang.org/x/net/http2"
)
func (b *BaseClient) createHttpClient() (*http.Client, error) {
func (b *BaseClient) createHttpClient() (http_client *http.Client) {
// Optimized connection pool settings for better performance
httpTransport := &http.Transport{
MaxIdleConns: 100, // Reduced from 512
@@ -26,14 +24,7 @@ func (b *BaseClient) createHttpClient() (*http.Client, error) {
ReadBufferSize: 4096, // Optimize buffer size
}
var http_client *http.Client
// Protect read of endpoint field
b.mu.RLock()
endpoint := b.endpoint
b.mu.RUnlock()
if strings.HasPrefix(endpoint, "http://") {
if strings.HasPrefix(b.endpoint, "http://") {
http_client = &http.Client{
Timeout: 30 * time.Second, // Increased for better reliability
Transport: httpTransport,
@@ -45,12 +36,10 @@ func (b *BaseClient) createHttpClient() (*http.Client, error) {
Message: "Using HTTP/1.1 over http",
Pairs: nil,
})
} else if strings.HasPrefix(endpoint, "https://") {
tlsClientConfig := getHTTPClientTLSConfig()
if strings.HasPrefix(endpoint, "https://") && tlsClientConfig.InsecureSkipVerify {
b.Logger.Warning(&libpack_logger.LogMessage{
Message: "TLS certificate verification is disabled. This is insecure and should not be used in production.",
})
} else if strings.HasPrefix(b.endpoint, "https://") {
tlsClientConfig := &tls.Config{}
if strings.HasPrefix(b.endpoint, "https://") {
tlsClientConfig.InsecureSkipVerify = true
}
http2Transport := &http2.Transport{
AllowHTTP: true,
@@ -75,25 +64,7 @@ func (b *BaseClient) createHttpClient() (*http.Client, error) {
b.Logger.Critical(&libpack_logger.LogMessage{
Message: "Invalid endpoint - neither http or https",
})
// Return error instead of nil to prevent nil pointer dereference
return nil, fmt.Errorf("invalid endpoint: %s - must start with http:// or https://", endpoint)
}
return http_client, nil
}
// getHTTPClientTLSConfig returns TLS configuration for HTTP client
// By default, TLS verification is enabled for security.
// Set GRAPHQL_INSECURE_SKIP_VERIFY=true to disable verification (NOT RECOMMENDED for production)
func getHTTPClientTLSConfig() *tls.Config {
skipVerify := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY") == "true"
if skipVerify {
// Log warning to stderr when running in insecure mode
fmt.Fprintf(os.Stderr, "WARNING: TLS certificate verification is disabled. This is insecure and should not be used in production.\n")
}
return &tls.Config{
InsecureSkipVerify: skipVerify,
MinVersion: tls.VersionTLS12, // Enforce minimum TLS 1.2 for security
return nil
}
return http_client
}
+13 -88
View File
@@ -2,7 +2,6 @@ package gql
import (
"net/http"
"os"
"testing"
"time"
@@ -14,8 +13,7 @@ func (suite *Tests) TestBaseClient_createHttpClient() {
client := NewConnection()
client.SetEndpoint("http://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
httpClient := client.createHttpClient()
assert.NotNil(httpClient)
assert.Equal(30*time.Second, httpClient.Timeout)
assert.NotNil(httpClient.Transport)
@@ -35,22 +33,10 @@ func (suite *Tests) TestBaseClient_createHttpClient() {
})
suite.T().Run("should create HTTP/2 client for https endpoints", func(t *testing.T) {
// Set environment variable for test to skip TLS verification
oldValue := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY")
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", "true")
t.Cleanup(func() {
if oldValue != "" {
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", oldValue)
} else {
os.Unsetenv("GRAPHQL_INSECURE_SKIP_VERIFY")
}
})
client := NewConnection()
client.SetEndpoint("https://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
httpClient := client.createHttpClient()
assert.NotNil(httpClient)
assert.Equal(30*time.Second, httpClient.Timeout)
assert.NotNil(httpClient.Transport)
@@ -70,9 +56,7 @@ func (suite *Tests) TestBaseClient_createHttpClient() {
client := NewConnection()
client.SetEndpoint("ftp://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.Error(err)
assert.Contains(err.Error(), "invalid endpoint")
httpClient := client.createHttpClient()
assert.Nil(httpClient)
})
@@ -80,24 +64,21 @@ func (suite *Tests) TestBaseClient_createHttpClient() {
client := NewConnection()
client.SetEndpoint("http://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
httpClient := client.createHttpClient()
assert.NotNil(httpClient)
assert.NotNil(httpClient.CheckRedirect)
// Test redirect policy
req, _ := http.NewRequest("GET", "http://example.com", nil)
err2 := httpClient.CheckRedirect(req, []*http.Request{})
assert.Equal(http.ErrUseLastResponse, err2)
err := httpClient.CheckRedirect(req, []*http.Request{})
assert.Equal(http.ErrUseLastResponse, err)
})
suite.T().Run("should handle empty endpoint", func(t *testing.T) {
client := NewConnection()
client.SetEndpoint("")
httpClient, err := client.createHttpClient()
assert.Error(err)
assert.Contains(err.Error(), "invalid endpoint")
httpClient := client.createHttpClient()
assert.Nil(httpClient)
})
@@ -105,35 +86,9 @@ func (suite *Tests) TestBaseClient_createHttpClient() {
client := NewConnection()
client.SetEndpoint("example.com/graphql")
httpClient, err := client.createHttpClient()
assert.Error(err)
assert.Contains(err.Error(), "invalid endpoint")
httpClient := client.createHttpClient()
assert.Nil(httpClient)
})
suite.T().Run("should enable TLS verification by default for https endpoints", func(t *testing.T) {
// Ensure the environment variable is not set
oldValue := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY")
os.Unsetenv("GRAPHQL_INSECURE_SKIP_VERIFY")
t.Cleanup(func() {
if oldValue != "" {
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", oldValue)
}
})
client := NewConnection()
client.SetEndpoint("https://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
assert.NotNil(httpClient)
// Check that TLS verification is enabled by default
transport, ok := httpClient.Transport.(*http2.Transport)
assert.True(ok)
assert.NotNil(transport.TLSClientConfig)
assert.False(transport.TLSClientConfig.InsecureSkipVerify) // Should be false by default
})
}
func (suite *Tests) TestBaseClient_createHttpClient_transportSettings() {
@@ -141,9 +96,7 @@ func (suite *Tests) TestBaseClient_createHttpClient_transportSettings() {
client := NewConnection()
client.SetEndpoint("http://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
assert.NotNil(httpClient)
httpClient := client.createHttpClient()
transport := httpClient.Transport.(*http.Transport)
// Verify all transport settings
@@ -159,23 +112,10 @@ func (suite *Tests) TestBaseClient_createHttpClient_transportSettings() {
})
suite.T().Run("should configure HTTP/2 transport correctly", func(t *testing.T) {
// Set environment variable for test to skip TLS verification
oldValue := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY")
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", "true")
t.Cleanup(func() {
if oldValue != "" {
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", oldValue)
} else {
os.Unsetenv("GRAPHQL_INSECURE_SKIP_VERIFY")
}
})
client := NewConnection()
client.SetEndpoint("https://example.com/graphql")
httpClient, err := client.createHttpClient()
assert.NoError(err)
assert.NotNil(httpClient)
httpClient := client.createHttpClient()
transport := httpClient.Transport.(*http2.Transport)
// Verify all HTTP/2 transport settings
@@ -195,30 +135,17 @@ func (suite *Tests) TestBaseClient_createHttpClient_logging() {
// This test verifies that the function runs without error
// The actual logging is tested indirectly through the logger tests
httpClient, err := client.createHttpClient()
assert.NoError(err)
httpClient := client.createHttpClient()
assert.NotNil(httpClient)
})
suite.T().Run("should log HTTP/2 usage", func(t *testing.T) {
// Set environment variable for test to skip TLS verification
oldValue := os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY")
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", "true")
t.Cleanup(func() {
if oldValue != "" {
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", oldValue)
} else {
os.Unsetenv("GRAPHQL_INSECURE_SKIP_VERIFY")
}
})
client := NewConnection()
client.SetEndpoint("https://example.com/graphql")
// This test verifies that the function runs without error
// The actual logging is tested indirectly through the logger tests
httpClient, err := client.createHttpClient()
assert.NoError(err)
httpClient := client.createHttpClient()
assert.NotNil(httpClient)
})
@@ -228,9 +155,7 @@ func (suite *Tests) TestBaseClient_createHttpClient_logging() {
// This test verifies that the function runs without error
// and returns nil for invalid endpoints (Critical() won't exit in tests)
httpClient, err := client.createHttpClient()
assert.Error(err)
assert.Contains(err.Error(), "invalid endpoint")
httpClient := client.createHttpClient()
assert.Nil(httpClient)
})
}
-2
View File
@@ -10,7 +10,6 @@ import (
// TestQueryJSONEncoding verifies that the Query struct serializes correctly
// without including the JsonQuery field, preventing "Trailing garbage" errors
func TestQueryJSONEncoding(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
client := CreateTestClient()
@@ -48,7 +47,6 @@ func TestQueryJSONEncoding(t *testing.T) {
// TestQueryJSONEncodingPreventsTrailingGarbage ensures the fix prevents the specific error
func TestQueryJSONEncodingPreventsTrailingGarbage(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
client := CreateTestClient()
+32 -51
View File
@@ -1,8 +1,8 @@
package gql
import (
"fmt"
"net/http"
"strings"
"time"
"github.com/gookit/goutil/envutil"
@@ -10,6 +10,22 @@ import (
logging "github.com/lukaszraczylo/go-simple-graphql/logging"
)
// parseRetryPatterns parses a comma-separated string of retry patterns into a slice
func parseRetryPatterns(patterns string) []string {
if patterns == "" {
return []string{}
}
parts := strings.Split(patterns, ",")
result := make([]string, 0, len(parts))
for _, p := range parts {
trimmed := strings.TrimSpace(p)
if trimmed != "" {
result = append(result, trimmed)
}
}
return result
}
func NewConnection() (b *BaseClient) {
// Read LOG_LEVEL environment variable
logLevelStr := envutil.Getenv("LOG_LEVEL", "info")
@@ -31,28 +47,18 @@ func NewConnection() (b *BaseClient) {
})
b = &BaseClient{
endpoint: envutil.Getenv("GRAPHQL_ENDPOINT", "https://api.github.com/graphql"),
responseType: envutil.Getenv("GRAPHQL_OUTPUT", "string"),
Logger: logger,
cache: cache.New(time.Duration(envutil.GetInt("GRAPHQL_CACHE_TTL", 5)) * time.Second),
cache_global: envutil.GetBool("GRAPHQL_CACHE_ENABLED", false),
retries_enable: envutil.GetBool("GRAPHQL_RETRIES_ENABLE", false),
retries_delay: time.Duration(envutil.GetInt("GRAPHQL_RETRIES_DELAY", 250) * int(time.Millisecond)),
retries_number: envutil.GetInt("GRAPHQL_RETRIES_NUMBER", 3),
minify_queries: envutil.GetBool("GRAPHQL_MINIFY_QUERIES", true), // Default: enabled for production efficiency
endpoint: envutil.Getenv("GRAPHQL_ENDPOINT", "https://api.github.com/graphql"),
responseType: envutil.Getenv("GRAPHQL_OUTPUT", "string"),
Logger: logger,
cache: cache.New(time.Duration(envutil.GetInt("GRAPHQL_CACHE_TTL", 5)) * time.Second),
cache_global: envutil.GetBool("GRAPHQL_CACHE_ENABLED", false),
retries_enable: envutil.GetBool("GRAPHQL_RETRIES_ENABLE", false),
retries_delay: time.Duration(envutil.GetInt("GRAPHQL_RETRIES_DELAY", 250) * int(time.Millisecond)),
retries_number: envutil.GetInt("GRAPHQL_RETRIES_NUMBER", 3),
retries_patterns: parseRetryPatterns(envutil.Getenv("GRAPHQL_RETRIES_PATTERNS", "postgres,connection,timeout,transaction,could not,temporarily unavailable,deadlock")),
minify_queries: envutil.GetBool("GRAPHQL_MINIFY_QUERIES", true), // Default: enabled for production efficiency
}
client, err := b.createHttpClient()
if err != nil {
b.Logger.Critical(&logging.LogMessage{
Message: "Failed to create HTTP client",
Pairs: map[string]interface{}{
"error": err.Error(),
},
})
// For backward compatibility, we'll still return the client but it won't work
return b
}
b.client = client
b.client = b.createHttpClient()
b.Logger.Debug(&logging.LogMessage{
Message: "Created new GraphQL client connection",
Pairs: map[string]interface{}{
@@ -63,46 +69,21 @@ func NewConnection() (b *BaseClient) {
}
func (b *BaseClient) SetEndpoint(endpoint string) {
b.mu.Lock()
defer b.mu.Unlock()
b.endpoint = endpoint
}
func (b *BaseClient) SetOutput(responseType string) error {
// Validate responseType - allowed values are "byte", "string", "mapstring"
allowedTypes := map[string]bool{
"byte": true,
"string": true,
"mapstring": true,
}
if !allowedTypes[responseType] {
return fmt.Errorf("invalid response type: %s. Allowed values are: byte, string, mapstring", responseType)
}
b.mu.Lock()
defer b.mu.Unlock()
func (b *BaseClient) SetOutput(responseType string) {
// allowed are byte, string, mapstring
// check if responseType is allowed
// TODO: implement
b.responseType = responseType
b.Logger.Debug(&logging.LogMessage{
Message: "Response type updated",
Pairs: map[string]interface{}{
"response_type": responseType,
},
})
return nil
}
func (b *BaseClient) SetHTTPClient(client *http.Client) {
b.mu.Lock()
defer b.mu.Unlock()
b.client = client
}
func (b *BaseClient) SetQueryMinification(enabled bool) {
b.mu.Lock()
defer b.mu.Unlock()
b.minify_queries = enabled
b.Logger.Debug(&logging.LogMessage{
Message: "GraphQL query minification setting updated",
+6 -50
View File
@@ -10,13 +10,6 @@ import (
libpack_logger "github.com/lukaszraczylo/go-simple-graphql/logging"
)
// Size limit constants for security
const (
MaxQuerySize = 100 * 1024 // 100KB max query size
MaxVariableSize = 10 * 1024 // 10KB max variables size
MaxResponseSize = 10 * 1024 * 1024 // 10MB max response size
)
// minifyGraphQLQuery removes unnecessary whitespace from GraphQL queries while preserving string literals
func minifyGraphQLQuery(query string) string {
if query == "" {
@@ -245,10 +238,7 @@ func (b *BaseClient) compileQuery(queryPartials ...any) *Query {
// Apply query minification if enabled (default: true)
finalQuery := query
b.mu.RLock()
minifyEnabled := b.minify_queries
b.mu.RUnlock()
if minifyEnabled {
if b.minify_queries {
originalSize := len(query)
minifiedQuery := minifyGraphQLQuery(query)
minifiedSize := len(minifiedQuery)
@@ -278,36 +268,9 @@ func (b *BaseClient) compileQuery(queryPartials ...any) *Query {
}
func (b *BaseClient) Query(query string, variables map[string]interface{}, headers map[string]interface{}) (any, error) {
// Validate query size limit
if len(query) > MaxQuerySize {
b.Logger.Error(&libpack_logger.LogMessage{
Message: "Query exceeds maximum size limit",
Pairs: map[string]interface{}{
"query_size": len(query),
"max_size": MaxQuerySize,
},
})
return nil, fmt.Errorf("query size %d exceeds maximum allowed size of %d bytes", len(query), MaxQuerySize)
}
// Process flags before compilation to avoid recompilation
enableCache, enableRetries, cleanedVariables := processFlags(variables, headers)
// Validate variables size limit
if cleanedVariables != nil {
variablesJSON, err := json.Marshal(cleanedVariables)
if err == nil && len(variablesJSON) > MaxVariableSize {
b.Logger.Error(&libpack_logger.LogMessage{
Message: "Variables exceed maximum size limit",
Pairs: map[string]interface{}{
"variables_size": len(variablesJSON),
"max_size": MaxVariableSize,
},
})
return nil, fmt.Errorf("variables size %d exceeds maximum allowed size of %d bytes", len(variablesJSON), MaxVariableSize)
}
}
// Compile query once with cleaned variables
compiledQuery := b.compileQuery(query, cleanedVariables)
if compiledQuery == nil || compiledQuery.JsonQuery == nil {
@@ -319,14 +282,11 @@ func (b *BaseClient) Query(query string, variables map[string]interface{}, heade
}
b.Logger.Debug(&libpack_logger.LogMessage{
Message: "Compiled query",
Pairs: map[string]interface{}{"query": sanitizeForLogging(compiledQuery.Query)},
Pairs: map[string]interface{}{"query": compiledQuery},
})
var queryHash string
b.mu.RLock()
cacheGlobal := b.cache_global
b.mu.RUnlock()
if (enableCache || cacheGlobal) && strutil.HasPrefix(compiledQuery.Query, "query") {
if (enableCache || b.cache_global) && strutil.HasPrefix(compiledQuery.Query, "query") {
b.Logger.Debug(&libpack_logger.LogMessage{
Message: "Cache enabled",
Pairs: nil,
@@ -335,13 +295,13 @@ func (b *BaseClient) Query(query string, variables map[string]interface{}, heade
if cachedValue := b.cacheLookup(queryHash); cachedValue != nil {
b.Logger.Debug(&libpack_logger.LogMessage{
Message: "Cache hit",
Pairs: map[string]interface{}{"query_hash": queryHash},
Pairs: map[string]interface{}{"query": compiledQuery},
})
return b.decodeResponse(cachedValue)
}
b.Logger.Debug(&libpack_logger.LogMessage{
Message: "Cache miss",
Pairs: map[string]interface{}{"query_hash": queryHash},
Pairs: map[string]interface{}{"query": compiledQuery},
})
}
@@ -355,11 +315,7 @@ func (b *BaseClient) Query(query string, variables map[string]interface{}, heade
}
return "no-cache"
}(),
Retries: func() bool {
b.mu.RLock()
defer b.mu.RUnlock()
return enableRetries || b.retries_enable
}(),
Retries: enableRetries || b.retries_enable,
}
rv, err := q.executeQuery()
-17
View File
@@ -8,7 +8,6 @@ import (
)
func TestMinifyGraphQLQuery(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
tests := []struct {
@@ -132,7 +131,6 @@ func TestMinifyGraphQLQuery(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
result := minifyGraphQLQuery(tt.input)
assert.Equal(tt.expected, result, "Minified query should match expected output")
@@ -152,7 +150,6 @@ func TestMinifyGraphQLQuery(t *testing.T) {
}
func TestMinifyGraphQLQuerySizeReduction(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
// Test with a heavily formatted query
@@ -211,11 +208,9 @@ func TestMinifyGraphQLQuerySizeReduction(t *testing.T) {
}
func TestBaseClientQueryMinification(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
t.Run("should_minify_queries_by_default", func(t *testing.T) {
t.Parallel()
client := CreateTestClient()
// Verify minification is enabled by default
@@ -238,7 +233,6 @@ func TestBaseClientQueryMinification(t *testing.T) {
})
t.Run("should_respect_minification_disabled", func(t *testing.T) {
t.Parallel()
client := CreateTestClient()
client.SetQueryMinification(false)
@@ -260,7 +254,6 @@ func TestBaseClientQueryMinification(t *testing.T) {
})
t.Run("should_handle_string_literals_correctly", func(t *testing.T) {
t.Parallel()
client := CreateTestClient()
query := `mutation test($value: String!) {
@@ -285,7 +278,6 @@ func TestBaseClientQueryMinification(t *testing.T) {
})
t.Run("should_demonstrate_size_reduction_with_production_case", func(t *testing.T) {
t.Parallel()
client := CreateTestClient()
// Use the production case from trailing garbage test
@@ -332,7 +324,6 @@ func TestBaseClientQueryMinification(t *testing.T) {
}
func TestIsAlphaNumeric(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
tests := []struct {
@@ -360,7 +351,6 @@ func TestIsAlphaNumeric(t *testing.T) {
for _, tt := range tests {
t.Run(string(tt.char), func(t *testing.T) {
t.Parallel()
result := isAlphaNumeric(tt.char)
assert.Equal(tt.expected, result, "isAlphaNumeric should correctly identify character type")
})
@@ -368,11 +358,9 @@ func TestIsAlphaNumeric(t *testing.T) {
}
func TestQueryMinificationEnvironmentVariable(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
t.Run("should_respect_GRAPHQL_MINIFY_QUERIES_env_var", func(t *testing.T) {
t.Parallel()
// This test would require setting environment variables
// For now, we'll test the default behavior
client := NewConnection()
@@ -383,23 +371,19 @@ func TestQueryMinificationEnvironmentVariable(t *testing.T) {
}
func TestQueryMinificationEdgeCases(t *testing.T) {
t.Parallel()
assert := assertions.New(t)
t.Run("should_handle_empty_query", func(t *testing.T) {
t.Parallel()
result := minifyGraphQLQuery("")
assert.Equal("", result, "Empty query should remain empty")
})
t.Run("should_handle_query_with_only_whitespace", func(t *testing.T) {
t.Parallel()
result := minifyGraphQLQuery(" \n\t \n ")
assert.Equal("", result, "Query with only whitespace should become empty")
})
t.Run("should_handle_nested_quotes", func(t *testing.T) {
t.Parallel()
query := `mutation test {
insert_test(value: "String with \"nested quotes\" and 'mixed' quotes") {
id
@@ -411,7 +395,6 @@ func TestQueryMinificationEdgeCases(t *testing.T) {
})
t.Run("should_preserve_spaces_between_keywords", func(t *testing.T) {
t.Parallel()
query := `query test($var: String!) {
field(where: {id: {_eq: $var}}) {
id
+1 -2
View File
@@ -71,8 +71,7 @@ func (suite *RequestCompressionTestSuite) TestRequestCompression() {
client.endpoint = endpoint
// Create HTTP client using the same logic as the library
httpClient, err := client.createHttpClient()
assert.NoError(err, "HTTP client should be created without error")
httpClient := client.createHttpClient()
assert.NotNil(httpClient, "HTTP client should be created")
// Verify transport settings
-63
View File
@@ -1,63 +0,0 @@
package gql
import (
"regexp"
"strings"
)
// Sensitive data patterns to redact from logs
var (
// Pattern for common sensitive field names in JSON/GraphQL
sensitiveFieldPattern = regexp.MustCompile(`(?i)"?(password|passwd|pwd|token|apikey|api_key|api-key|secret|authorization|auth|bearer|credentials|private_key|private-key|access_token|refresh_token|client_secret|session|cookie)"?\s*:\s*"[^"]*"`)
// Pattern for authorization headers
authHeaderPattern = regexp.MustCompile(`(?i)(authorization|x-api-key|x-auth-token|bearer)\s*:\s*[^\s,}]+`)
// Pattern for basic auth in URLs
basicAuthURLPattern = regexp.MustCompile(`(https?://)([^:]+):([^@]+)@`)
// Pattern for JWT tokens
jwtPattern = regexp.MustCompile(`eyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+`)
)
const redactedText = "[REDACTED]"
// sanitizeForLogging removes sensitive data from strings before logging
// This includes passwords, tokens, API keys, authorization headers, etc.
func sanitizeForLogging(input string) string {
if input == "" {
return input
}
// Create a working copy
sanitized := input
// Redact sensitive fields in JSON/GraphQL
sanitized = sensitiveFieldPattern.ReplaceAllStringFunc(sanitized, func(match string) string {
// Preserve the field name but redact the value
parts := strings.SplitN(match, ":", 2)
if len(parts) == 2 {
fieldName := parts[0]
return fieldName + `: "` + redactedText + `"`
}
return redactedText
})
// Redact authorization headers
sanitized = authHeaderPattern.ReplaceAllStringFunc(sanitized, func(match string) string {
parts := strings.SplitN(match, ":", 2)
if len(parts) == 2 {
headerName := parts[0]
return headerName + ": " + redactedText
}
return redactedText
})
// Redact basic auth in URLs
sanitized = basicAuthURLPattern.ReplaceAllString(sanitized, "${1}"+redactedText+":"+redactedText+"@")
// Redact JWT tokens
sanitized = jwtPattern.ReplaceAllString(sanitized, redactedText)
return sanitized
}
+12 -13
View File
@@ -2,7 +2,6 @@ package gql
import (
"net/http"
"sync"
"time"
cache "github.com/lukaszraczylo/go-simple-graphql/cache"
@@ -10,18 +9,18 @@ import (
)
type BaseClient struct {
mu sync.RWMutex // Protects all fields for thread-safe access
cache *cache.Cache
Logger *logging.Logger
client *http.Client
endpoint string
responseType string
retries_delay time.Duration
retries_number int
MaxGoRoutines int
cache_global bool
retries_enable bool
minify_queries bool // Enable GraphQL query minification (default: true)
cache *cache.Cache
Logger *logging.Logger
client *http.Client
endpoint string
responseType string
retries_delay time.Duration
retries_number int
retries_patterns []string // Error patterns that should trigger retries
MaxGoRoutines int
cache_global bool
retries_enable bool
minify_queries bool // Enable GraphQL query minification (default: true)
}
type Query struct {
+2 -18
View File
@@ -2,7 +2,6 @@ package gql
import (
"net/http/httptest"
"os"
"testing"
assertions "github.com/stretchr/testify/assert"
@@ -14,19 +13,11 @@ type Tests struct {
}
var (
assert *assertions.Assertions
mockServer *httptest.Server
originalInsecureSkipValue string
assert *assertions.Assertions
mockServer *httptest.Server
)
func (suite *Tests) SetupSuite() {
// Save original environment variable value
originalInsecureSkipValue = os.Getenv("GRAPHQL_INSECURE_SKIP_VERIFY")
// Set environment variable to skip TLS verification for test mock server
// The mock server uses self-signed certificates
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", "true")
// Start mock GraphQL server
mockServer = StartMockServer()
}
@@ -39,13 +30,6 @@ func (suite *Tests) TearDownSuite() {
// Cleanup shared test cache
CleanupTestCache()
// Restore original environment variable value
if originalInsecureSkipValue != "" {
os.Setenv("GRAPHQL_INSECURE_SKIP_VERIFY", originalInsecureSkipValue)
} else {
os.Unsetenv("GRAPHQL_INSECURE_SKIP_VERIFY")
}
}
func (suite *Tests) SetupTest() {