Compare commits

...

5 Commits

Author SHA1 Message Date
lukaszraczylo b4c047819f Update dependencies. 2023-11-18 02:12:59 +00:00
lukaszraczylo 1390e7cdd1 Fix blocking the introspection + add unit tests. 2023-11-18 02:11:38 +00:00
lukaszraczylo a71b3950db Load retrospection query set once. 2023-11-17 22:32:58 +00:00
lukaszraczylo 827c26e88d Fix retrospection query blocking. 2023-11-17 22:29:42 +00:00
lukaszraczylo 30528e4a9a Sort labels by keys before pushing them to metrics registry.
This is needed to ensure that labels are always in the same order and
metrics won't produce duplicates.
2023-11-17 14:36:23 +00:00
7 changed files with 414 additions and 51 deletions
+1 -1
View File
@@ -11,7 +11,7 @@ help: ## display this help
.PHONY: run
run: build ## run application
@LOG_LEVEL=debug PURGE_METRICS_ON_CRAWL=true BLOCK_SCHEMA_INTROSPECTION=false CACHE_TTL=10 JWT_ROLE_RATE_LIMIT=false JWT_ROLE_CLAIM_PATH="Hasura.x-hasura-default-role" JWT_USER_CLAIM_PATH="Hasura.x-hasura-user-id" HOST_GRAPHQL=https://hasura8.lan/ HEALTHCHECK_GRAPHQL_URL=https://hasura8.lan/v1/graphql ./graphql-proxy
@LOG_LEVEL=debug PURGE_METRICS_ON_CRAWL=true BLOCK_SCHEMA_INTROSPECTION=true CACHE_TTL=10 JWT_ROLE_RATE_LIMIT=false JWT_ROLE_CLAIM_PATH="Hasura.x-hasura-default-role" JWT_USER_CLAIM_PATH="Hasura.x-hasura-user-id" HOST_GRAPHQL=https://hasura8.lan/ HEALTHCHECK_GRAPHQL_URL=https://hasura8.lan/v1/graphql ./graphql-proxy
.PHONY: build
build: ## build the binary
+3 -3
View File
@@ -16,15 +16,15 @@ require (
github.com/lukaszraczylo/go-simple-graphql v1.1.37
github.com/rs/zerolog v1.31.0
github.com/stretchr/testify v1.8.4
github.com/valyala/fasthttp v1.50.0
github.com/valyala/fasthttp v1.51.0
)
require (
github.com/andybalholm/brotli v1.0.6 // indirect
github.com/avast/retry-go/v4 v4.5.0 // indirect
github.com/avast/retry-go/v4 v4.5.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gookit/color v1.5.4 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/klauspost/compress v1.17.3 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
+6 -6
View File
@@ -2,8 +2,8 @@ github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aE
github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg=
github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I=
github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -27,8 +27,8 @@ github.com/graphql-go/graphql v0.8.1 h1:p7/Ou/WpmulocJeEx7wjQy611rtXGQaAcXGqanuM
github.com/graphql-go/graphql v0.8.1/go.mod h1:nKiHzRM0qopJEwCITUuIsxk9PlVlwIiiI8pnJEhordQ=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA=
github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@@ -74,8 +74,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M=
github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA=
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
+93 -32
View File
@@ -1,6 +1,7 @@
package main
import (
"flag"
"strconv"
"strings"
@@ -10,7 +11,7 @@ import (
libpack_monitoring "github.com/lukaszraczylo/graphql-monitoring-proxy/monitoring"
)
var retrospection_queries = []string{
var introspection_queries = []string{
"__schema",
"__type",
"__typename",
@@ -34,7 +35,29 @@ var retrospection_queries = []string{
}
// Saving the introspection queries as a map O(1) operation instead of O(n) for a slice.
var retrospectionQuerySet = make(map[string]struct{}, len(retrospection_queries))
var introspectionQuerySet = map[string]struct{}{}
var introspectionAllowedQueries = map[string]struct{}{}
func prepareQueriesAndExemptions() {
introspectionQuerySet = map[string]struct{}{}
introspectionQuerySet = func() map[string]struct{} {
rsqs := make(map[string]struct{}, len(introspection_queries))
for _, query := range introspection_queries {
rsqs[strings.ToLower(query)] = struct{}{}
}
return rsqs
}()
introspectionAllowedQueries = map[string]struct{}{}
introspectionAllowedQueries = func() map[string]struct{} {
rsqs := make(map[string]struct{}, len(cfg.Security.IntrospectionAllowed))
for _, query := range cfg.Security.IntrospectionAllowed {
rsqs[strings.ToLower(query)] = struct{}{}
}
return rsqs
}()
}
func parseGraphQLQuery(c *fiber.Ctx) (operationType, operationName string, cacheRequest bool, cache_time int, should_block bool, should_ignore bool) {
should_ignore = true
@@ -42,21 +65,27 @@ func parseGraphQLQuery(c *fiber.Ctx) (operationType, operationName string, cache
err := json.Unmarshal(c.Body(), &m)
if err != nil {
cfg.Logger.Debug("Can't unmarshal the request", map[string]interface{}{"error": err.Error(), "body": string(c.Body())})
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
}
return
}
// get the query
query, ok := m["query"].(string)
if !ok {
cfg.Logger.Error("Can't find the query", map[string]interface{}{"query": query, "m_val": m})
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
}
return
}
p, err := parser.Parse(parser.ParseParams{Source: query})
if err != nil {
cfg.Logger.Error("Can't parse the query", map[string]interface{}{"query": query, "m_val": m})
cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
}
return
}
@@ -65,19 +94,21 @@ func parseGraphQLQuery(c *fiber.Ctx) (operationType, operationName string, cache
for _, d := range p.Definitions {
if oper, ok := d.(*ast.OperationDefinition); ok {
operationType = oper.Operation
if oper.Name != nil {
operationName = oper.Name.Value
}
if strings.ToLower(operationType) == "mutation" && cfg.Server.ReadOnlyMode {
cfg.Logger.Warning("Mutation blocked", m)
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
}
c.Status(403).SendString("The server is in read-only mode")
should_block = true
return
}
if oper.Name != nil {
operationName = oper.Name.Value
} else {
operationName = "undefined"
}
for _, dir := range oper.Directives {
if dir.Name.Value == "cached" {
cacheRequest = true
@@ -85,37 +116,67 @@ func parseGraphQLQuery(c *fiber.Ctx) (operationType, operationName string, cache
if arg.Name.Value == "ttl" {
cache_time, err = strconv.Atoi(arg.Value.GetValue().(string))
if err != nil {
cfg.Logger.Error("Can't parse the ttl", map[string]interface{}{"ttl": arg.Value.GetValue().(string)})
cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
cfg.Logger.Error("Can't parse the ttl, using global", map[string]interface{}{"bad_ttl": arg.Value.GetValue().(string)})
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsFailed, nil)
}
return
}
}
}
}
}
if cfg.Security.BlockIntrospection {
for _, s := range oper.SelectionSet.Selections {
for _, s2 := range s.GetSelectionSet().Selections {
if _, exists := retrospectionQuerySet[strings.ToLower(s2.(*ast.Field).Name.Value)]; exists {
if len(cfg.Security.IntrospectionAllowed) > 0 {
for _, introspectionQueryAllowed := range cfg.Security.IntrospectionAllowed {
if strings.EqualFold(strings.ToLower(introspectionQueryAllowed), strings.ToLower(s2.(*ast.Field).Name.Value)) {
cfg.Logger.Debug("Introspection query allowed, passing through", m)
return
}
}
}
cfg.Logger.Warning("Introspection query blocked", m)
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
c.Status(403).SendString("Introspection queries are not allowed")
should_block = true
return
}
}
should_block = checkSelections(c, oper.GetSelectionSet().Selections)
if should_block {
return
}
}
}
}
return
}
func checkSelections(c *fiber.Ctx, selections []ast.Selection) bool {
for _, s := range selections {
field, ok := s.(*ast.Field)
if !ok {
continue // or handle the case where the type assertion fails
}
shouldBlock := checkIfContainsIntrospection(c, field.Name.Value)
if shouldBlock {
return true
}
if field.SelectionSet != nil {
if checkSelections(c, field.GetSelectionSet().Selections) {
return true
}
}
}
return false
}
func checkIfContainsIntrospection(c *fiber.Ctx, whatever string) (should_block bool) {
whateverLower := strings.ToLower(whatever)
got_exemption := false
if _, exists := introspectionQuerySet[whateverLower]; exists {
if len(cfg.Security.IntrospectionAllowed) > 0 {
if _, allowed_exists := introspectionAllowedQueries[whateverLower]; allowed_exists {
cfg.Logger.Debug("Introspection query allowed, passing through", map[string]interface{}{"query": whatever})
got_exemption = true
should_block = false
}
}
if !got_exemption {
should_block = true
}
}
if should_block {
if flag.Lookup("test.v") == nil {
cfg.Monitoring.Increment(libpack_monitoring.MetricsSkipped, nil)
}
c.Status(403).SendString("Introspection queries are not allowed")
}
return
}
+301
View File
@@ -0,0 +1,301 @@
package main
import (
"testing"
fiber "github.com/gofiber/fiber/v2"
libpack_logging "github.com/lukaszraczylo/graphql-monitoring-proxy/logging"
"github.com/valyala/fasthttp"
)
func (suite *Tests) Test_parseGraphQLQuery() {
type results struct {
is_cached bool
cached_ttl int
should_block bool
should_ignore bool
op_name string
op_type string
returnCode int
}
type queries struct {
body string
headers map[string]string
}
tests := []struct {
name string
suppliedSettings *config
suppliedQuery queries
wantResults results
}{
{
name: "test empty body",
suppliedQuery: queries{
body: "",
headers: map[string]string{},
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: true,
op_name: "",
op_type: "",
},
},
{
name: "test empty json",
suppliedQuery: queries{
body: "{}",
headers: map[string]string{},
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: true,
op_name: "",
op_type: "",
},
},
{
name: "test empty with some random garbage",
suppliedQuery: queries{
body: "{\"variables\": {\"id\": \"1\"}}",
headers: map[string]string{},
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: true,
op_name: "",
op_type: "",
},
},
{
name: "test valid query with op name",
suppliedQuery: queries{
body: "{\"query\":\"query MyQuery { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\"}",
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: false,
op_name: "MyQuery",
op_type: "query",
},
},
{
name: "test valid query with op name, variables and cache",
suppliedQuery: queries{
body: "{\"query\":\"query MyQuery @cached { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\", \"variables\": {\"id\": \"1\"}}",
},
wantResults: results{
is_cached: true,
should_block: false,
should_ignore: false,
op_name: "MyQuery",
op_type: "query",
},
},
{
name: "test valid query with op name, cache and ttl",
suppliedQuery: queries{
body: "{\"query\":\"query MyQuery @cached(ttl: 60) { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\", \"variables\": {\"id\": \"1\"}}",
},
wantResults: results{
is_cached: true,
cached_ttl: 60,
should_block: false,
should_ignore: false,
op_name: "MyQuery",
op_type: "query",
},
},
{
name: "test valid query with op name, cache and INVALID ttl",
suppliedQuery: queries{
body: "{\"query\":\"query MyQuery @cached(ttl: nope) { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\", \"variables\": {\"id\": \"1\"}}",
},
wantResults: results{
is_cached: true,
cached_ttl: 0,
should_block: false,
should_ignore: false,
op_name: "MyQuery",
op_type: "query",
},
},
{
name: "test mutation query with op name",
suppliedQuery: queries{
body: "{\"query\":\"mutation MyMutation { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\"}",
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: false,
op_name: "MyMutation",
op_type: "mutation",
},
},
{
name: "test mutation query with config: read only",
suppliedSettings: func() *config {
cfg.Server.ReadOnlyMode = true
return cfg
}(),
suppliedQuery: queries{
body: "{\"query\":\"mutation MyMutation { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __typename } }\"}",
},
wantResults: results{
is_cached: false,
should_block: true,
should_ignore: false,
op_name: "MyMutation",
op_type: "mutation",
returnCode: 403,
},
},
{
name: "test simple query with introspection __schema",
suppliedQuery: queries{
body: "{\"query\":\"mutation MyMutation { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __schema } }\"}",
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: false,
op_name: "MyMutation",
op_type: "mutation",
},
},
{
name: "test simple query with introspection __schema config: block introspection",
suppliedSettings: func() *config {
cfg.Security.BlockIntrospection = true
return cfg
}(),
suppliedQuery: queries{
body: "{\"query\":\"query MyIntroQuery { tg_users(where: {handle: {_eq: \\\"tozuo\\\"}}) { id __schema } }\"}",
},
wantResults: results{
is_cached: false,
should_block: true,
should_ignore: false,
op_name: "MyIntroQuery",
op_type: "query",
returnCode: 403,
},
},
{
name: "test user supplied query with introspection #1 - config: block",
suppliedSettings: func() *config {
parseConfig()
cfg.Security.BlockIntrospection = true
cfg.Security.IntrospectionAllowed = []string{}
prepareQueriesAndExemptions()
return cfg
}(),
suppliedQuery: queries{
body: "{\"query\":\"{__schema {queryType {fields {name description}}}}\"}",
},
wantResults: results{
is_cached: false,
should_block: true,
should_ignore: false,
op_name: "undefined",
op_type: "query",
returnCode: 403,
},
},
{
name: "test user supplied query with introspection #1 - config: block & allow __schema",
suppliedSettings: func() *config {
parseConfig()
cfg.Security.BlockIntrospection = true
cfg.Security.IntrospectionAllowed = []string{"__schema"}
prepareQueriesAndExemptions()
return cfg
}(),
suppliedQuery: queries{
body: "{\"query\":\"{__schema {queryType {fields {name description}}}}\"}",
},
wantResults: results{
is_cached: false,
should_block: false,
should_ignore: false,
op_name: "undefined",
op_type: "query",
returnCode: 200,
},
},
}
for _, tt := range tests {
suite.T().Run(tt.name, func(t *testing.T) {
cfg = &config{}
cfg.Logger = libpack_logging.NewLogger()
defer func() {
cfg = &config{}
}()
app := fiber.New()
ctx_headers := func() *fasthttp.RequestHeader {
h := fasthttp.RequestHeader{}
for k, v := range tt.suppliedQuery.headers {
h.Add(k, v)
}
return &h
}()
ctx_request := fasthttp.Request{
Header: *ctx_headers,
}
ctx_request.AppendBody([]byte(tt.suppliedQuery.body))
ctx := app.AcquireCtx(&fasthttp.RequestCtx{
Request: ctx_request,
})
defer app.ReleaseCtx(ctx)
assert.NotNil(ctx, "Fiber context is nil")
if tt.suppliedSettings != nil {
cfg = tt.suppliedSettings
}
defer func() {
cfg = &config{}
}()
opType, opName, cacheFromQuery, cached_ttl, shouldBlock, should_ignore := parseGraphQLQuery(ctx)
assert.Equal(tt.wantResults.op_type, opType, "Unexpected operation type", tt.name)
assert.Equal(tt.wantResults.op_name, opName, "Unexpected operation name", tt.name)
assert.Equal(tt.wantResults.is_cached, cacheFromQuery, "Unexpected cache value", tt.name)
assert.Equal(tt.wantResults.cached_ttl, cached_ttl, "Unexpected cache TTL value", tt.name)
assert.Equal(tt.wantResults.should_block, shouldBlock, "Unexpected block value", tt.name)
assert.Equal(tt.wantResults.should_ignore, should_ignore, "Unexpected ignore value", tt.name)
if tt.wantResults.returnCode > 0 {
assert.Equal(tt.wantResults.returnCode, ctx.Response().StatusCode(), "Unexpected return code", tt.name)
}
})
}
}
+2 -7
View File
@@ -11,15 +11,9 @@ import (
var cfg *config
func init() {
for _, query := range retrospection_queries {
retrospectionQuerySet[query] = struct{}{}
}
}
func parseConfig() {
libpack_config.PKG_NAME = "graphql_proxy"
var c config
c := config{}
c.Server.PortGraphQL = envutil.GetInt("PORT_GRAPHQL", 8080)
c.Server.PortMonitoring = envutil.GetInt("MONITORING_PORT", 9393)
c.Server.HostGraphQL = envutil.Getenv("HOST_GRAPHQL", "http://localhost/")
@@ -61,6 +55,7 @@ func parseConfig() {
enableCache() // takes close to no resources, but can be used with dynamic query cache
loadRatelimitConfig()
enableApi()
prepareQueriesAndExemptions()
}
func main() {
+8 -2
View File
@@ -3,6 +3,7 @@ package libpack_monitoring
import (
"fmt"
"os"
"sort"
"strings"
libpack_config "github.com/lukaszraczylo/graphql-monitoring-proxy/config"
@@ -25,9 +26,14 @@ func (ms *MetricsSetup) get_metrics_name(name string, labels map[string]string)
complete_name = name
}
if labels != nil {
keys := make([]string, 0, len(labels))
for k := range labels {
keys = append(keys, k)
}
sort.Strings(keys)
complete_name += "{"
for k, v := range labels {
complete_name += k + "=\"" + v + "\","
for _, k := range keys {
complete_name += k + "=\"" + labels[k] + "\","
}
complete_name = strings.TrimSuffix(complete_name, ",")
complete_name += "}"