llm-testbench/internal/tests/codebase_analysis_test.go

package tests

import (
	"context"
	"testing"
)

func TestCodeBugLineDiffTest_Eval(t *testing.T) {
	tc := codeBugLineDiffTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct: root-cause line", `{"line":8}`, 1},
		// B9: a numeric-string JSON value must coerce the same as a
		// native number - the prompt asked for the right line, not a
		// specific JSON type.
		{"correct as a numeric string", `{"line":"8"}`, 1},
		{"wrong: points at the panic site instead of the root cause", `{"line":10}`, 0},
		{"wrong line entirely", `{"line":6}`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeRaceVariableTest_Eval(t *testing.T) {
	tc := codeRaceVariableTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct", `{"variable":"total"}`, 1},
		{"correct different case", `{"variable":"Total"}`, 1},
		{"wrong: loop variable, not the shared counter", `{"variable":"i"}`, 0},
		{"wrong: function parameter, not the shared counter", `{"variable":"n"}`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeDeadFunctionsTest_Eval(t *testing.T) {
	tc := codeDeadFunctionsTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct set", `["helperC","helperD"]`, 1},
		{"correct set, different order", `["helperD","helperC"]`, 1},
		{"missing one dead function", `["helperD"]`, 0.5},
		{"wrongly includes a live function", `["helperC","helperD","helperB"]`, 2.0 / 3.0},
		{"wrong: says nothing is dead", `[]`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeBigODedupTest_Eval(t *testing.T) {
	tc := codeBigODedupTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"caret form", "O(n^2)", 1},
		{"double-star form", "O(n**2)", 1},
		{"unicode superscript form", "O(n²)", 1},
		{"n*n form", "O(n*n)", 1},
		{"lowercase o", "o(n^2)", 1},
		{"quoted", `"O(n^2)"`, 1},
		{"trailing period", "O(n^2).", 1},
		{"wrong: linear", "O(n)", 0},
		{"wrong: linearithmic", "O(n log n)", 0},
		// B5: the correct notation appears as a substring of a negated
		// sentence; the whole-response-anchored evaluator must not match
		// on that substring.
		{"negated: contains the substring O(n^2) but names O(n) as the answer", "not O(n^2), it's O(n) with a hash set", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeImportCycleTest_Eval(t *testing.T) {
	tc := codeImportCycleTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct set", `["a","b","c"]`, 1},
		{"correct set, different order", `["c","a","b"]`, 1},
		{"wrongly includes non-cyclic d", `["a","b","c","d"]`, 0.75},
		{"missing one cycle member", `["a","b"]`, 2.0 / 3.0},
		{"wrong: only d, which has no imports at all", `["d"]`, 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeGenericReturnTypeTest_Eval(t *testing.T) {
	tc := codeGenericReturnTypeTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct", "string", 1},
		{"correct capitalized", "String", 1},
		{"correct quoted", "`string`", 1},
		{"wrong: the slice element pointer type", "*string", 0},
		{"wrong: the generic parameter name itself, not its instantiation", "T", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestCodeCommitBisectTest_Eval(t *testing.T) {
	tc := codeCommitBisectTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{"correct", "c3c3c3c", 1},
		{"correct with trailing period", "c3c3c3c.", 1},
		{"wrong: the initial commit, predates the cache entirely", "c1a1a1a", 0},
		{"wrong: only a logging change, no behavior change", "c4d4d4d", 0},
		{"wrong: the docs-only commit", "c5e5e5e", 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}