llm-testbench/internal/eval/jsonfield_test.go

package eval

import (
	"context"
	"testing"
)

func TestExtractJSON(t *testing.T) {
	tests := []struct {
		name     string
		response string
		want     string
		wantErr  bool
	}{
		{"plain object", `{"bump":"minor"}`, `{"bump":"minor"}`, false},
		{"fenced object", "Here you go:\n```json\n{\"bump\":\"minor\"}\n```", `{"bump":"minor"}`, false},
		{"object with prose around", `The answer is {"bump":"minor"} as computed.`, `{"bump":"minor"}`, false},
		{"nested braces ignored inside strings", `{"note":"a { b } c","bump":"patch"}`, `{"note":"a { b } c","bump":"patch"}`, false},
		{"array", `["ClaudeBot","Googlebot"]`, `["ClaudeBot","Googlebot"]`, false},
		{"no json", "no json here at all", "", true},
		// S7: a candidate JSON considered during reasoning, followed by the
		// real final JSON, must resolve to the final one.
		{
			name:     "reasoning with a candidate JSON then a final JSON: final wins",
			response: `Let me think. One option would be {"bump":"major"} but that is not right. The actual answer is {"bump":"minor"}.`,
			want:     `{"bump":"minor"}`,
			wantErr:  false,
		},
		{
			name:     "fenced json block preferred over any other json-looking text before it",
			response: "Draft: {\"bump\":\"major\"}\n\nFinal answer:\n```json\n{\"bump\":\"minor\"}\n```",
			want:     `{"bump":"minor"}`,
			wantErr:  false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got, err := ExtractJSON(tt.response)
			if (err != nil) != tt.wantErr {
				t.Fatalf("ExtractJSON(%q) error = %v, wantErr %v", tt.response, err, tt.wantErr)
			}
			if err == nil && got != tt.want {
				t.Errorf("ExtractJSON(%q) = %q, want %q", tt.response, got, tt.want)
			}
		})
	}
}

func TestJSONField_String(t *testing.T) {
	e := JSONField("bump", "minor")
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"exact match", `{"bump":"minor"}`, 1},
		{"case insensitive", `{"bump":"MINOR"}`, 1},
		{"wrong value", `{"bump":"patch"}`, 0},
		{"missing field", `{"other":"x"}`, 0},
		{"fenced json", "```json\n{\"bump\": \"minor\"}\n```", 1},
		{"invalid json", "not json at all", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestJSONField_Int(t *testing.T) {
	e := JSONField("M", 16)
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"exact match", `{"M":16,"efConstruction":200,"efSearch":50}`, 1},
		{"wrong value", `{"M":32}`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestJSONField_Int_NumericStringCoercion is the B9 regression: a model
// answering {"line":"8"} instead of {"line":8} must still score 1 - the
// prompt asked for a JSON field with the right value, not a specific JSON
// type.
func TestJSONField_Int_NumericStringCoercion(t *testing.T) {
	e := JSONField("line", 8)
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"numeric string coerces", `{"line":"8"}`, 1},
		{"numeric string with whitespace coerces", `{"line":" 8 "}`, 1},
		{"wrong numeric string", `{"line":"9"}`, 0},
		{"non-numeric string does not coerce", `{"line":"eight"}`, 0},
		{"native number still works", `{"line":8}`, 1},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestJSONField_Float_NumericStringCoercion mirrors the int case for a
// float64-typed field.
func TestJSONField_Float_NumericStringCoercion(t *testing.T) {
	e := JSONField("port", float64(8080))
	got := e.Evaluate(context.Background(), `{"port":"8080"}`)
	if got.Value != 1 {
		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
	}
}

// TestJSONField_Bool_StringCoercion is the DC2 regression, mirroring B9's
// numeric-string coercion: a model answering {"guaranteed_visible":"false"}
// instead of {"guaranteed_visible":false} must still score 1.
func TestJSONField_Bool_StringCoercion(t *testing.T) {
	e := JSONField("guaranteed_visible", false)
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"bool string coerces (lowercase)", `{"guaranteed_visible":"false"}`, 1},
		{"bool string coerces with whitespace", `{"guaranteed_visible":" false "}`, 1},
		{"bool string coerces, different case", `{"guaranteed_visible":"False"}`, 1},
		{"wrong bool string", `{"guaranteed_visible":"true"}`, 0},
		{"non-boolean string does not coerce", `{"guaranteed_visible":"nope"}`, 0},
		{"native bool still works", `{"guaranteed_visible":false}`, 1},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestJSONField_EnumHyphenSpaceFold is the DC4 regression: a multi-word
// enum answer spelled with spaces instead of hyphens (or vice versa) must
// still score 1.
func TestJSONField_EnumHyphenSpaceFold(t *testing.T) {
	e := JSONField("anomaly", "non-repeatable-read")
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"exact hyphenated form", `{"anomaly":"non-repeatable-read"}`, 1},
		{"space-separated form folds equal (DC4 bug probe)", `{"anomaly":"non repeatable read"}`, 1},
		{"mixed hyphen/space form folds equal (DC4 bug probe)", `{"anomaly":"non repeatable-read"}`, 1},
		{"a genuinely different enum value still fails", `{"anomaly":"dirty-read"}`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestJSONField_NestedPath(t *testing.T) {
	e := JSONField("task1", "search_web")
	got := e.Evaluate(context.Background(), `{"task1":"search_web","task2":"read_file"}`)
	if got.Value != 1 {
		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
	}
}

func TestJSONField_IndexedPath(t *testing.T) {
	e := JSONField("steps[0]", "build")
	got := e.Evaluate(context.Background(), `{"steps":["build","test","deploy"]}`)
	if got.Value != 1 {
		t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail)
	}
	got = e.Evaluate(context.Background(), `{"steps":["test","build","deploy"]}`)
	if got.Value != 0 {
		t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail)
	}
}

func TestJSONStringSet(t *testing.T) {
	e := JSONStringSet([]string{"ClaudeBot", "Googlebot"})
	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"exact set", `["ClaudeBot","Googlebot"]`, 1},
		{"exact set different order", `["Googlebot","ClaudeBot"]`, 1},
		{"case insensitive", `["claudebot","googlebot"]`, 1},
		{"missing one", `["ClaudeBot"]`, 0.5},
		{"extra one", `["ClaudeBot","Googlebot","GPTBot"]`, float64(2) / 3},
		{"completely wrong", `["GPTBot","PerplexityBot"]`, 0},
		{"invalid json", "no array here", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestJSONStringArrayEquals(t *testing.T) {
	want := []string{"build", "test", "backup", "deploy", "verify"}
	e := JSONStringArrayEquals(want)

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"exact order", `["build","test","backup","deploy","verify"]`, 1},
		{"case insensitive exact order", `["BUILD","TEST","BACKUP","DEPLOY","VERIFY"]`, 1},
		{"wrong order", `["build","backup","test","deploy","verify"]`, 0},
		{"wrong length", `["build","test"]`, 0},
		{"invalid json", "not an array", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := e.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}