From 85cca6ec920a96662954c6a348747a43164c23da Mon Sep 17 00:00:00 2001 From: Lukasz Raczylo Date: Fri, 23 May 2025 13:32:31 +0100 Subject: [PATCH] fixup! fixup! Library improvements. --- execute_query.go | 122 +++++++++++++++--- execute_query_test.go | 286 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+), 17 deletions(-) diff --git a/execute_query.go b/execute_query.go index f8a58b1..be901be 100644 --- a/execute_query.go +++ b/execute_query.go @@ -21,7 +21,7 @@ var ( MaxIdleConns: 100, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, - DisableCompression: false, // Enable compression for responses + DisableCompression: true, // Disable automatic compression to handle gzip manually ForceAttemptHTTP2: true, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // Skip TLS verification for test environments } @@ -58,6 +58,11 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) { httpRequest.Header.Set("Content-Type", "application/json") } + // Set Accept-Encoding header to request gzip compression + if httpRequest.Header.Get("Accept-Encoding") == "" { + httpRequest.Header.Set("Accept-Encoding", "gzip") + } + retriesMax := 1 if qe.Retries { retriesMax = qe.retries_number @@ -92,32 +97,115 @@ func (qe *QueryExecutor) executeQuery() ([]byte, error) { return fmt.Errorf("HTTP error - status code: %s for %s", httpResponse.Status, httpRequest.URL) } - var reader io.Reader + // Log Content-Encoding header for debugging encoding := httpResponse.Header.Get("Content-Encoding") - if encoding == "gzip" { - gzipReader, err := gzip.NewReader(httpResponse.Body) - if err != nil { - return fmt.Errorf("error creating gzip reader: %w", err) - } - defer gzipReader.Close() - reader = gzipReader - } else { - reader = httpResponse.Body + qe.Logger.Debug(&libpack_logger.LogMessage{ + Message: "Processing response", + Pairs: map[string]interface{}{ + "content_encoding": encoding, + "content_type": httpResponse.Header.Get("Content-Type"), + }, + }) + + // First, read the entire response body into a buffer + rawBuf := bufferPool.Get().(*bytes.Buffer) + rawBuf.Reset() + defer bufferPool.Put(rawBuf) + + _, err = io.Copy(rawBuf, httpResponse.Body) + if err != nil { + return fmt.Errorf("error reading HTTP response body: %w", err) } - // Use buffer pool for reading response + rawData := rawBuf.Bytes() + qe.Logger.Debug(&libpack_logger.LogMessage{ + Message: "Read raw response data", + Pairs: map[string]interface{}{ + "response_size": len(rawData), + }, + }) + + // Use buffer pool for final data processing respBuf := bufferPool.Get().(*bytes.Buffer) respBuf.Reset() defer bufferPool.Put(respBuf) - _, err = io.Copy(respBuf, reader) - if err != nil { - return fmt.Errorf("error reading HTTP response: %w", err) + var finalData []byte + + // Handle gzip decompression with fallback + if encoding == "gzip" { + // Try to create gzip reader from raw data + gzipReader, err := gzip.NewReader(bytes.NewReader(rawData)) + if err != nil { + qe.Logger.Warning(&libpack_logger.LogMessage{ + Message: "Failed to create gzip reader, falling back to uncompressed parsing", + Pairs: map[string]interface{}{"gzip_error": err.Error()}, + }) + // Fallback: treat raw data as uncompressed + finalData = rawData + } else { + // Successfully created gzip reader, now try to decompress + defer gzipReader.Close() + + _, copyErr := io.Copy(respBuf, gzipReader) + if copyErr != nil { + // Gzip decompression failed, try fallback + qe.Logger.Warning(&libpack_logger.LogMessage{ + Message: "Gzip decompression failed, falling back to uncompressed parsing", + Pairs: map[string]interface{}{ + "copy_error": copyErr, + "raw_size": len(rawData), + }, + }) + // Fallback: treat raw data as uncompressed + finalData = rawData + } else { + // Successful decompression + finalData = respBuf.Bytes() + qe.Logger.Debug(&libpack_logger.LogMessage{ + Message: "Successfully decompressed gzip response", + Pairs: map[string]interface{}{ + "compressed_size": len(rawData), + "decompressed_size": len(finalData), + }, + }) + } + } + } else { + // No compression, use raw data directly + finalData = rawData } - // Unmarshal response directly from buffer - err = json.Unmarshal(respBuf.Bytes(), &queryResult) + // Validate that we have data before attempting JSON unmarshaling + if len(finalData) == 0 { + return fmt.Errorf("empty response data after processing") + } + + // Log the first few bytes for debugging (truncated for safety) + debugData := finalData + if len(debugData) > 100 { + debugData = debugData[:100] + } + qe.Logger.Debug(&libpack_logger.LogMessage{ + Message: "Attempting JSON unmarshaling", + Pairs: map[string]interface{}{ + "data_size": len(finalData), + "data_sample": string(debugData), + }, + }) + + // Unmarshal the final processed data + err = json.Unmarshal(finalData, &queryResult) if err != nil { + qe.Logger.Error(&libpack_logger.LogMessage{ + Message: "JSON unmarshaling failed", + Pairs: map[string]interface{}{ + "error": err.Error(), + "data_size": len(finalData), + "data_sample": string(debugData), + "encoding": encoding, + }, + }) return fmt.Errorf("error unmarshalling HTTP response: %w", err) } diff --git a/execute_query_test.go b/execute_query_test.go index 35bc1bf..7d1586a 100644 --- a/execute_query_test.go +++ b/execute_query_test.go @@ -306,3 +306,289 @@ func (suite *Tests) TestQueryExecutor_executeQuery_requestCreationError() { assert.Contains(err.Error(), "can't create HTTP request") }) } + +func (suite *Tests) TestQueryExecutor_executeQuery_gzipTrailingGarbage() { + suite.T().Run("should handle gzip trailing garbage error with fallback", func(t *testing.T) { + // Create a test server that returns malformed gzip data (with trailing garbage) + malformedGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + // Create valid gzip data first + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + gzipWriter.Write([]byte(`{"data":{"viewer":{"login":"testuser"}}}`)) + gzipWriter.Close() + + // Add trailing garbage to simulate the error + validGzipData := buf.Bytes() + malformedData := append(validGzipData, []byte("trailing garbage data")...) + + w.WriteHeader(http.StatusOK) + w.Write(malformedData) + })) + defer malformedGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = malformedGzipServer.URL + qe.client = malformedGzipServer.Client() + qe.cache = client.cache + + // This should fail with gzip decompression but fallback to treating as uncompressed + result, err := qe.executeQuery() + // The fallback should fail because the raw data (including gzip headers) is not valid JSON + assert.Error(err) + assert.Nil(result) + assert.Contains(err.Error(), "error unmarshalling HTTP response") + }) + + suite.T().Run("should handle gzip with valid JSON fallback", func(t *testing.T) { + // Create a test server that claims gzip but returns plain JSON + fakeGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") // Claims gzip but sends plain JSON + + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"data":{"viewer":{"login":"fallbackuser"}}}`)) + })) + defer fakeGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = fakeGzipServer.URL + qe.client = fakeGzipServer.Client() + qe.cache = client.cache + + // Should fail gzip decompression but succeed with fallback to plain JSON + result, err := qe.executeQuery() + assert.NoError(err) + assert.NotNil(result) + assert.Contains(string(result), "fallbackuser") + }) + + suite.T().Run("should handle corrupted gzip header", func(t *testing.T) { + // Create a test server that returns corrupted gzip data + corruptedGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + // Send corrupted gzip data (invalid magic number) + corruptedData := []byte{0x1f, 0x8c, 0x08, 0x00} // Invalid gzip header + corruptedData = append(corruptedData, []byte(`{"data":{"viewer":{"login":"corruptuser"}}}`)...) + + w.WriteHeader(http.StatusOK) + w.Write(corruptedData) + })) + defer corruptedGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = corruptedGzipServer.URL + qe.client = corruptedGzipServer.Client() + qe.cache = client.cache + + // Should fail gzip reader creation and fallback to raw data parsing + result, err := qe.executeQuery() + // This will likely fail because the raw data includes corrupted gzip headers + assert.Error(err) + assert.Nil(result) + }) + + suite.T().Run("should handle empty gzip response", func(t *testing.T) { + // Create a test server that returns empty gzip data + emptyGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + w.WriteHeader(http.StatusOK) + // Send empty response + })) + defer emptyGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = emptyGzipServer.URL + qe.client = emptyGzipServer.Client() + qe.cache = client.cache + + result, err := qe.executeQuery() + assert.Error(err) + assert.Nil(result) + assert.Contains(err.Error(), "empty response data") + }) + + suite.T().Run("should handle partial gzip data", func(t *testing.T) { + // Create a test server that returns partial/truncated gzip data + partialGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + // Create valid gzip data but truncate it + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + gzipWriter.Write([]byte(`{"data":{"viewer":{"login":"partialuser"}}}`)) + gzipWriter.Close() + + // Send only partial gzip data (truncated) + fullData := buf.Bytes() + partialData := fullData[:len(fullData)/2] // Send only half + + w.WriteHeader(http.StatusOK) + w.Write(partialData) + })) + defer partialGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = partialGzipServer.URL + qe.client = partialGzipServer.Client() + qe.cache = client.cache + + result, err := qe.executeQuery() + // Should fail gzip decompression and fallback, but raw data won't be valid JSON + assert.Error(err) + assert.Nil(result) + }) + + suite.T().Run("should successfully decompress valid gzip", func(t *testing.T) { + // Ensure our fix doesn't break valid gzip handling + validGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + gzipWriter.Write([]byte(`{"data":{"viewer":{"login":"validgzipuser"}}}`)) + gzipWriter.Close() + + w.WriteHeader(http.StatusOK) + w.Write(buf.Bytes()) + })) + defer validGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "test-valid-gzip", + Retries: false, + } + qe.endpoint = validGzipServer.URL + qe.client = validGzipServer.Client() + qe.cache = client.cache + qe.CacheTTL = 5 * time.Second + + result, err := qe.executeQuery() + assert.NoError(err) + assert.NotNil(result) + assert.Contains(string(result), "validgzipuser") + }) +} + +func (suite *Tests) TestQueryExecutor_executeQuery_gzipErrorScenarios() { + suite.T().Run("should handle gzip reader creation failure", func(t *testing.T) { + // Test server that sends invalid gzip magic number + invalidMagicServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + // Send data with wrong magic number (not gzip) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"data":{"viewer":{"login":"magicuser"}}}`)) + })) + defer invalidMagicServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = invalidMagicServer.URL + qe.client = invalidMagicServer.Client() + qe.cache = client.cache + + result, err := qe.executeQuery() + // Should fail gzip reader creation, fallback to raw JSON parsing, and succeed + assert.NoError(err) + assert.NotNil(result) + assert.Contains(string(result), "magicuser") + }) + + suite.T().Run("should handle gzip decompression with unexpected EOF", func(t *testing.T) { + // Create server that sends truncated gzip stream + eofGzipServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Encoding", "gzip") + + // Create a proper gzip header but truncate the data + gzipHeader := []byte{0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff} + // Add some compressed data but not complete + incompleteData := []byte{0x4a, 0x49, 0x2c, 0x49, 0x54, 0xb2, 0x52, 0x50} + + w.WriteHeader(http.StatusOK) + w.Write(append(gzipHeader, incompleteData...)) + })) + defer eofGzipServer.Close() + + client := CreateTestClient() + + qe := &QueryExecutor{ + BaseClient: client, + Query: []byte(`{"query":"query { viewer { login } }"}`), + Headers: map[string]interface{}{"Content-Type": "application/json"}, + CacheKey: "no-cache", + Retries: false, + } + qe.endpoint = eofGzipServer.URL + qe.client = eofGzipServer.Client() + qe.cache = client.cache + + result, err := qe.executeQuery() + // Should fail gzip decompression and fallback, but raw data won't be valid JSON + assert.Error(err) + assert.Nil(result) + assert.Contains(err.Error(), "error unmarshalling HTTP response") + }) +}