llm-testbench/internal/tests/agents_planning_test.go

package tests

import (
	"context"
	"reflect"
	"testing"
)

// planEnumerateTopoOrders returns every permutation of steps' ids that is a
// valid topological order (every step appears after all of its Deps). It
// exists only to prove, by exhaustive brute-force enumeration, that a
// planning fixture admits exactly one valid order (or, for planCycleSteps,
// admits none) - the small N (<=5) used by every fixture in this file
// makes brute force cheap.
func planEnumerateTopoOrders(steps []planStep) [][]string {
	ids := make([]string, len(steps))
	depsByID := make(map[string][]string, len(steps))
	for i, s := range steps {
		ids[i] = s.ID
		depsByID[s.ID] = s.Deps
	}

	var permutations [][]string
	var permute func(remaining, chosen []string)
	permute = func(remaining, chosen []string) {
		if len(remaining) == 0 {
			order := make([]string, len(chosen))
			copy(order, chosen)
			permutations = append(permutations, order)
			return
		}
		for i, id := range remaining {
			rest := make([]string, 0, len(remaining)-1)
			rest = append(rest, remaining[:i]...)
			rest = append(rest, remaining[i+1:]...)
			nextChosen := make([]string, len(chosen)+1)
			copy(nextChosen, chosen)
			nextChosen[len(chosen)] = id
			permute(rest, nextChosen)
		}
	}
	permute(ids, nil)

	var valid [][]string
	for _, order := range permutations {
		position := make(map[string]int, len(order))
		for i, id := range order {
			position[id] = i
		}
		ok := true
		for id, deps := range depsByID {
			for _, dep := range deps {
				if position[dep] > position[id] {
					ok = false
					break
				}
			}
			if !ok {
				break
			}
		}
		if ok {
			valid = append(valid, order)
		}
	}
	return valid
}

// planIsPriorityConsistent independently re-derives, at every position in
// order, the "ready" set (steps whose Deps are already placed earlier) and
// checks the chosen step is the minimum by (priority, id) among them - the
// same rule planResourceConstrainedTest's prompt states in words. It is
// used to filter planEnumerateTopoOrders' output down to the orders
// consistent with that rule, independently of planScheduleSingleWorker.
func planIsPriorityConsistent(order []string, steps []planStep, priority map[string]int) bool {
	depsByID := make(map[string][]string, len(steps))
	for _, s := range steps {
		depsByID[s.ID] = s.Deps
	}
	done := make(map[string]bool, len(steps))
	for _, chosen := range order {
		var ready []string
		for _, s := range steps {
			if done[s.ID] {
				continue
			}
			allDone := true
			for _, dep := range depsByID[s.ID] {
				if !done[dep] {
					allDone = false
					break
				}
			}
			if allDone {
				ready = append(ready, s.ID)
			}
		}
		best := ready[0]
		for _, id := range ready[1:] {
			if priority[id] < priority[best] || (priority[id] == priority[best] && id < best) {
				best = id
			}
		}
		if chosen != best {
			return false
		}
		done[chosen] = true
	}
	return true
}

func TestAgentPlanOrderingTest_Eval(t *testing.T) {
	tc := agentPlanOrderingTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{
			name:     "correct order",
			response: `["build","test","backup","deploy","verify","rollback"]`,
			want:     1,
		},
		{
			name:     "correct order fenced",
			response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
			want:     1,
		},
		{
			name:     "backup before test violates dependency",
			response: `["build","backup","test","deploy","verify","rollback"]`,
			want:     0,
		},
		{
			name:     "rollback before verify",
			response: `["build","test","backup","deploy","rollback","verify"]`,
			want:     0,
		},
		{
			name:     "missing a step",
			response: `["build","test","backup","deploy","verify"]`,
			want:     0,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestAgentPlanOrderingTest_UniqueOrder confirms, by brute-force
// enumeration over all 6! permutations, that agentPlanOrderingWant is the
// only ordering of the 6 steps consistent with the prompt's pairwise
// constraints.
func TestAgentPlanOrderingTest_UniqueOrder(t *testing.T) {
	steps := []planStep{
		{ID: "build"},
		{ID: "test", Deps: []string{"build"}},
		{ID: "backup", Deps: []string{"test"}},
		{ID: "deploy", Deps: []string{"backup"}},
		{ID: "verify", Deps: []string{"deploy"}},
		{ID: "rollback", Deps: []string{"verify"}},
	}
	valid := planEnumerateTopoOrders(steps)
	if len(valid) != 1 {
		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
	}
	if !reflect.DeepEqual(valid[0], agentPlanOrderingWant) {
		t.Errorf("unique valid order = %v, want %v", valid[0], agentPlanOrderingWant)
	}
}

func TestPlanCriticalPathTest_Eval(t *testing.T) {
	tc := planCriticalPathTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "correct number", response: "12", want: 1},
		{name: "correct number with trailing reasoning line", response: "The critical path is build+test+package+deploy+verify.\n12", want: 1},
		{name: "correct number with unit suffix", response: "12 minutes", want: 1},
		{name: "wrong: shorter branch length", response: "11", want: 0},
		{name: "wrong: sum of all durations ignoring parallelism", response: "17", want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanCriticalPathWant_HandDerived cross-checks planCriticalPathLength's
// output against an independently hand-computed value (see the ground
// truth comment on planCriticalPathWant), rather than only comparing the
// algorithm to itself.
func TestPlanCriticalPathWant_HandDerived(t *testing.T) {
	const handDerived = 3 + 4 + 1 + 2 + 2 // build+test+package+deploy+verify
	if planCriticalPathWant != handDerived {
		t.Errorf("planCriticalPathWant = %d, hand-derived critical path = %d", planCriticalPathWant, handDerived)
	}
}

func TestPlanParallelStepsTest_Eval(t *testing.T) {
	tc := planParallelStepsTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "correct stages", response: `[["a","b"],["c","d"],["e"]]`, want: 1},
		{name: "correct stages, reordered within a stage", response: `[["b","a"],["d","c"],["e"]]`, want: 1},
		{name: "correct stages, fenced", response: "```json\n[[\"a\",\"b\"],[\"c\",\"d\"],[\"e\"]]\n```", want: 1},
		{name: "wrong: c and d not grouped together", response: `[["a","b"],["c"],["d"],["e"]]`, want: 0},
		{name: "wrong: e placed too early", response: `[["a","b","e"],["c","d"]]`, want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanWaveWant_HandDerived cross-checks planWaveLayers' output against
// an independently hand-computed layering.
func TestPlanWaveWant_HandDerived(t *testing.T) {
	want := [][]string{{"a", "b"}, {"c", "d"}, {"e"}}
	if !reflect.DeepEqual(planWaveWant, want) {
		t.Errorf("planWaveWant = %v, hand-derived = %v", planWaveWant, want)
	}
}

func TestPlanRollbackTriggerTest_Eval(t *testing.T) {
	tc := planRollbackTriggerTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "all correct", response: `{"trigger_after":"verify","condition":"verify_failure"}`, want: 1},
		{name: "all correct fenced", response: "```json\n{\"trigger_after\":\"verify\",\"condition\":\"verify_failure\"}\n```", want: 1},
		{name: "wrong trigger step", response: `{"trigger_after":"deploy","condition":"verify_failure"}`, want: 0.5},
		{name: "wrong condition", response: `{"trigger_after":"verify","condition":"deploy_failure"}`, want: 0.5},
		{name: "both wrong", response: `{"trigger_after":"test","condition":"manual_request"}`, want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestPlanIdempotencyTest_Eval(t *testing.T) {
	tc := planIdempotencyTest()

	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
	// out of a denominator of 3" (computed, not hardcoded, to avoid a
	// float-literal precision mismatch against the evaluator's own
	// float64(matched)/float64(denom) division).
	twoOfThree := float64(2) / float64(3)

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{
			name:     "exact correct set",
			response: `["fetch_deps","run_migrations","apply_terraform"]`,
			want:     1,
		},
		{
			name:     "exact correct set, different order",
			response: `["apply_terraform","fetch_deps","run_migrations"]`,
			want:     1,
		},
		{
			name:     "exact correct set, fenced",
			response: "```json\n[\"fetch_deps\",\"run_migrations\",\"apply_terraform\"]\n```",
			want:     1,
		},
		{
			name:     "includes an unsafe step",
			response: `["fetch_deps","run_migrations","apply_terraform","send_notification"]`,
			want:     0.75,
		},
		{
			name:     "missing a safe step",
			response: `["fetch_deps","run_migrations"]`,
			want:     twoOfThree,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestPlanRepairTest_Eval(t *testing.T) {
	tc := planRepairTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "correct: fix_and_rerun_test", response: `{"next_action":"fix_and_rerun_test"}`, want: 1},
		{name: "correct, fenced with prose", response: "The right move:\n```json\n{\"next_action\":\"fix_and_rerun_test\"}\n```", want: 1},
		{name: "correct, uppercase", response: `{"next_action":"FIX_AND_RERUN_TEST"}`, want: 1},
		{name: "wrong: rollback (nothing deployed yet)", response: `{"next_action":"rollback"}`, want: 0},
		{name: "wrong: proceed_to_backup (test never passed)", response: `{"next_action":"proceed_to_backup"}`, want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

func TestPlanMilestoneDateTest_Eval(t *testing.T) {
	tc := planMilestoneDateTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "correct number", response: "7", want: 1},
		{name: "correct number with trailing reasoning line", response: "prep_data finishes on day 5, M starts then and takes 2 more days.\n7", want: 1},
		{name: "correct number with unit suffix", response: "day 7", want: 1},
		{name: "wrong: used prep_env instead of the slower prerequisite", response: "5", want: 0},
		{name: "wrong: summed durations serially", response: "10", want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanMilestoneWant_HandDerived cross-checks planMilestoneWant against
// an independently hand-computed value.
func TestPlanMilestoneWant_HandDerived(t *testing.T) {
	const handDerived = 7
	if planMilestoneWant != handDerived {
		t.Errorf("planMilestoneWant = %d, hand-derived = %d", planMilestoneWant, handDerived)
	}
}

func TestPlanResourceConstrainedTest_Eval(t *testing.T) {
	tc := planResourceConstrainedTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{
			name:     "correct order",
			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model","smoke_test"]`,
			want:     1,
		},
		{
			name:     "correct order fenced",
			response: "```json\n[\"provision_ct\",\"install_graphann\",\"configure_network\",\"deploy_reranker_model\",\"smoke_test\"]\n```",
			want:     1,
		},
		{
			name:     "wrong: ignores priority tie-break at the first choice",
			response: `["configure_network","provision_ct","install_graphann","deploy_reranker_model","smoke_test"]`,
			want:     0,
		},
		{
			name:     "wrong: runs two steps at once conceptually (deploy before configure_network)",
			response: `["provision_ct","install_graphann","deploy_reranker_model","configure_network","smoke_test"]`,
			want:     0,
		},
		{
			name:     "missing a step",
			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model"]`,
			want:     0,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanResourceConstrainedTest_UniqueOrder independently proves, by
// brute-force enumeration over all 5! permutations filtered against the
// priority rule stated in the prompt, that planResourceConstrainedWant is
// the ONLY order consistent with both the dependency graph and the
// priority tie-break rule.
func TestPlanResourceConstrainedTest_UniqueOrder(t *testing.T) {
	all := planEnumerateTopoOrders(planResourceConstrainedSteps)

	var consistent [][]string
	for _, order := range all {
		if planIsPriorityConsistent(order, planResourceConstrainedSteps, planResourceConstrainedPriority) {
			consistent = append(consistent, order)
		}
	}
	if len(consistent) != 1 {
		t.Fatalf("expected exactly 1 priority-consistent topological order, found %d: %v", len(consistent), consistent)
	}
	if !reflect.DeepEqual(consistent[0], planResourceConstrainedWant) {
		t.Errorf("unique consistent order = %v, want %v", consistent[0], planResourceConstrainedWant)
	}
}

func TestPlanCycleDetectTest_Eval(t *testing.T) {
	tc := planCycleDetectTest()

	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
	// out of a denominator of 3" (computed, not hardcoded, to avoid a
	// float-literal precision mismatch against the evaluator's own
	// float64(matched)/float64(denom) division).
	twoOfThree := float64(2) / float64(3)

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "exact correct set", response: `["c","d","e"]`, want: 1},
		{name: "exact correct set, different order", response: `["e","c","d"]`, want: 1},
		{name: "exact correct set, fenced", response: "```json\n[\"c\",\"d\",\"e\"]\n```", want: 1},
		{name: "includes an acyclic task", response: `["a","c","d","e"]`, want: 0.75},
		{name: "wrong: only two of the three cyclic tasks", response: `["c","d"]`, want: twoOfThree},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanCycleDetectTest_NoValidOrder independently confirms
// planCycleSteps genuinely contains a cycle (the full graph admits zero
// valid topological orders under brute-force enumeration), and that
// removing the c-depends-on-e edge specifically restores at least one
// valid order - pinpointing that edge, and thus c/d/e, as the cycle.
func TestPlanCycleDetectTest_NoValidOrder(t *testing.T) {
	if got := planEnumerateTopoOrders(planCycleSteps); len(got) != 0 {
		t.Fatalf("expected zero valid topological orders for a graph containing a cycle, found %d: %v", len(got), got)
	}

	acyclic := make([]planStep, len(planCycleSteps))
	copy(acyclic, planCycleSteps)
	for i, s := range acyclic {
		if s.ID == "c" {
			acyclic[i] = planStep{ID: "c", Deps: []string{"b"}} // drop c's dependency on e
		}
	}
	if got := planEnumerateTopoOrders(acyclic); len(got) == 0 {
		t.Fatal("removing the c-depends-on-e edge should leave at least one valid topological order, found none")
	}
}

func TestPlanMinimalReplanTest_Eval(t *testing.T) {
	tc := planMinimalReplanTest()

	tests := []struct {
		name     string
		response string
		want     float64
	}{
		{name: "correct order", response: `["build","test","deploy","verify","rollback"]`, want: 1},
		{name: "correct order fenced", response: "```json\n[\"build\",\"test\",\"deploy\",\"verify\",\"rollback\"]\n```", want: 1},
		{name: "wrong: backup left in", response: `["build","test","backup","deploy","verify","rollback"]`, want: 0},
		{name: "wrong: rollback before verify", response: `["build","test","deploy","rollback","verify"]`, want: 0},
		{name: "missing a step", response: `["build","test","deploy","verify"]`, want: 0},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := tc.Eval.Evaluate(context.Background(), tt.response)
			if got.Value != tt.want {
				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
			}
		})
	}
}

// TestPlanMinimalReplanWant_UniqueOrder confirms, by brute-force
// enumeration, that planMinimalReplanWant is the only ordering of the 5
// remaining steps consistent with the requirement change's constraints.
func TestPlanMinimalReplanWant_UniqueOrder(t *testing.T) {
	steps := []planStep{
		{ID: "build"},
		{ID: "test", Deps: []string{"build"}},
		{ID: "deploy", Deps: []string{"test"}},
		{ID: "verify", Deps: []string{"deploy"}},
		{ID: "rollback", Deps: []string{"verify"}},
	}
	valid := planEnumerateTopoOrders(steps)
	if len(valid) != 1 {
		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
	}
	if !reflect.DeepEqual(valid[0], planMinimalReplanWant) {
		t.Errorf("unique valid order = %v, want %v", valid[0], planMinimalReplanWant)
	}
}