test(catalog): expand agents to 10 tests per subcategory

Bring agents/tool-routing and agents/planning up to 10 tests each. Add a new agents/delegation subcategory with 10 tests. Split agents.go into agents_toolrouting.go, agents_planning.go, and agents_delegation.go, each with a sibling _test.go file, and reduce agents.go to a thin registration aggregator. Every new prompt is self-contained and forces a JSON or number-only answer. Set-valued answers use JSONStringSet or a custom stage-set evaluator so no arbitrary order is required. Dependency-graph fixtures that force a single valid order are verified unique in the unit tests by brute-force enumeration over every permutation. Numeric ground truth is derived by running the same algorithm the fixture describes, then cross-checked against an independently hand-derived value. MaxTokens is omitted on all 28 new tests per the expansion rules; the 2 existing tests keep their prior value unchanged.
2026-10-12 00:45:19 +00:00 · 2026-08-17 01:47:36 +01:00
parent 764fe990fd
commit 053eeab1bb
8 changed files with 2689 additions and 173 deletions
@@ -0,0 +1,537 @@
+package tests
+
+import (
+	"context"
+	"reflect"
+	"testing"
+)
+
+// planEnumerateTopoOrders returns every permutation of steps' ids that is a
+// valid topological order (every step appears after all of its Deps). It
+// exists only to prove, by exhaustive brute-force enumeration, that a
+// planning fixture admits exactly one valid order (or, for planCycleSteps,
+// admits none) - the small N (<=5) used by every fixture in this file
+// makes brute force cheap.
+func planEnumerateTopoOrders(steps []planStep) [][]string {
+	ids := make([]string, len(steps))
+	depsByID := make(map[string][]string, len(steps))
+	for i, s := range steps {
+		ids[i] = s.ID
+		depsByID[s.ID] = s.Deps
+	}
+
+	var permutations [][]string
+	var permute func(remaining, chosen []string)
+	permute = func(remaining, chosen []string) {
+		if len(remaining) == 0 {
+			order := make([]string, len(chosen))
+			copy(order, chosen)
+			permutations = append(permutations, order)
+			return
+		}
+		for i, id := range remaining {
+			rest := make([]string, 0, len(remaining)-1)
+			rest = append(rest, remaining[:i]...)
+			rest = append(rest, remaining[i+1:]...)
+			nextChosen := make([]string, len(chosen)+1)
+			copy(nextChosen, chosen)
+			nextChosen[len(chosen)] = id
+			permute(rest, nextChosen)
+		}
+	}
+	permute(ids, nil)
+
+	var valid [][]string
+	for _, order := range permutations {
+		position := make(map[string]int, len(order))
+		for i, id := range order {
+			position[id] = i
+		}
+		ok := true
+		for id, deps := range depsByID {
+			for _, dep := range deps {
+				if position[dep] > position[id] {
+					ok = false
+					break
+				}
+			}
+			if !ok {
+				break
+			}
+		}
+		if ok {
+			valid = append(valid, order)
+		}
+	}
+	return valid
+}
+
+// planIsPriorityConsistent independently re-derives, at every position in
+// order, the "ready" set (steps whose Deps are already placed earlier) and
+// checks the chosen step is the minimum by (priority, id) among them - the
+// same rule planResourceConstrainedTest's prompt states in words. It is
+// used to filter planEnumerateTopoOrders' output down to the orders
+// consistent with that rule, independently of planScheduleSingleWorker.
+func planIsPriorityConsistent(order []string, steps []planStep, priority map[string]int) bool {
+	depsByID := make(map[string][]string, len(steps))
+	for _, s := range steps {
+		depsByID[s.ID] = s.Deps
+	}
+	done := make(map[string]bool, len(steps))
+	for _, chosen := range order {
+		var ready []string
+		for _, s := range steps {
+			if done[s.ID] {
+				continue
+			}
+			allDone := true
+			for _, dep := range depsByID[s.ID] {
+				if !done[dep] {
+					allDone = false
+					break
+				}
+			}
+			if allDone {
+				ready = append(ready, s.ID)
+			}
+		}
+		best := ready[0]
+		for _, id := range ready[1:] {
+			if priority[id] < priority[best] || (priority[id] == priority[best] && id < best) {
+				best = id
+			}
+		}
+		if chosen != best {
+			return false
+		}
+		done[chosen] = true
+	}
+	return true
+}
+
+func TestAgentPlanOrderingTest_Eval(t *testing.T) {
+	tc := agentPlanOrderingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct order",
+			response: `["build","test","backup","deploy","verify","rollback"]`,
+			want:     1,
+		},
+		{
+			name:     "correct order fenced",
+			response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "backup before test violates dependency",
+			response: `["build","backup","test","deploy","verify","rollback"]`,
+			want:     0,
+		},
+		{
+			name:     "rollback before verify",
+			response: `["build","test","backup","deploy","rollback","verify"]`,
+			want:     0,
+		},
+		{
+			name:     "missing a step",
+			response: `["build","test","backup","deploy","verify"]`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestAgentPlanOrderingTest_UniqueOrder confirms, by brute-force
+// enumeration over all 6! permutations, that agentPlanOrderingWant is the
+// only ordering of the 6 steps consistent with the prompt's pairwise
+// constraints.
+func TestAgentPlanOrderingTest_UniqueOrder(t *testing.T) {
+	steps := []planStep{
+		{ID: "build"},
+		{ID: "test", Deps: []string{"build"}},
+		{ID: "backup", Deps: []string{"test"}},
+		{ID: "deploy", Deps: []string{"backup"}},
+		{ID: "verify", Deps: []string{"deploy"}},
+		{ID: "rollback", Deps: []string{"verify"}},
+	}
+	valid := planEnumerateTopoOrders(steps)
+	if len(valid) != 1 {
+		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
+	}
+	if !reflect.DeepEqual(valid[0], agentPlanOrderingWant) {
+		t.Errorf("unique valid order = %v, want %v", valid[0], agentPlanOrderingWant)
+	}
+}
+
+func TestPlanCriticalPathTest_Eval(t *testing.T) {
+	tc := planCriticalPathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct number", response: "12", want: 1},
+		{name: "correct number with trailing reasoning line", response: "The critical path is build+test+package+deploy+verify.\n12", want: 1},
+		{name: "correct number with unit suffix", response: "12 minutes", want: 1},
+		{name: "wrong: shorter branch length", response: "11", want: 0},
+		{name: "wrong: sum of all durations ignoring parallelism", response: "17", want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanCriticalPathWant_HandDerived cross-checks planCriticalPathLength's
+// output against an independently hand-computed value (see the ground
+// truth comment on planCriticalPathWant), rather than only comparing the
+// algorithm to itself.
+func TestPlanCriticalPathWant_HandDerived(t *testing.T) {
+	const handDerived = 3 + 4 + 1 + 2 + 2 // build+test+package+deploy+verify
+	if planCriticalPathWant != handDerived {
+		t.Errorf("planCriticalPathWant = %d, hand-derived critical path = %d", planCriticalPathWant, handDerived)
+	}
+}
+
+func TestPlanParallelStepsTest_Eval(t *testing.T) {
+	tc := planParallelStepsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct stages", response: `[["a","b"],["c","d"],["e"]]`, want: 1},
+		{name: "correct stages, reordered within a stage", response: `[["b","a"],["d","c"],["e"]]`, want: 1},
+		{name: "correct stages, fenced", response: "```json\n[[\"a\",\"b\"],[\"c\",\"d\"],[\"e\"]]\n```", want: 1},
+		{name: "wrong: c and d not grouped together", response: `[["a","b"],["c"],["d"],["e"]]`, want: 0},
+		{name: "wrong: e placed too early", response: `[["a","b","e"],["c","d"]]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanWaveWant_HandDerived cross-checks planWaveLayers' output against
+// an independently hand-computed layering.
+func TestPlanWaveWant_HandDerived(t *testing.T) {
+	want := [][]string{{"a", "b"}, {"c", "d"}, {"e"}}
+	if !reflect.DeepEqual(planWaveWant, want) {
+		t.Errorf("planWaveWant = %v, hand-derived = %v", planWaveWant, want)
+	}
+}
+
+func TestPlanRollbackTriggerTest_Eval(t *testing.T) {
+	tc := planRollbackTriggerTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"trigger_after":"verify","condition":"verify_failure"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"trigger_after\":\"verify\",\"condition\":\"verify_failure\"}\n```", want: 1},
+		{name: "wrong trigger step", response: `{"trigger_after":"deploy","condition":"verify_failure"}`, want: 0.5},
+		{name: "wrong condition", response: `{"trigger_after":"verify","condition":"deploy_failure"}`, want: 0.5},
+		{name: "both wrong", response: `{"trigger_after":"test","condition":"manual_request"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanIdempotencyTest_Eval(t *testing.T) {
+	tc := planIdempotencyTest()
+
+	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
+	// out of a denominator of 3" (computed, not hardcoded, to avoid a
+	// float-literal precision mismatch against the evaluator's own
+	// float64(matched)/float64(denom) division).
+	twoOfThree := float64(2) / float64(3)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "exact correct set",
+			response: `["fetch_deps","run_migrations","apply_terraform"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, different order",
+			response: `["apply_terraform","fetch_deps","run_migrations"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, fenced",
+			response: "```json\n[\"fetch_deps\",\"run_migrations\",\"apply_terraform\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "includes an unsafe step",
+			response: `["fetch_deps","run_migrations","apply_terraform","send_notification"]`,
+			want:     0.75,
+		},
+		{
+			name:     "missing a safe step",
+			response: `["fetch_deps","run_migrations"]`,
+			want:     twoOfThree,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanRepairTest_Eval(t *testing.T) {
+	tc := planRepairTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: fix_and_rerun_test", response: `{"next_action":"fix_and_rerun_test"}`, want: 1},
+		{name: "correct, fenced with prose", response: "The right move:\n```json\n{\"next_action\":\"fix_and_rerun_test\"}\n```", want: 1},
+		{name: "correct, uppercase", response: `{"next_action":"FIX_AND_RERUN_TEST"}`, want: 1},
+		{name: "wrong: rollback (nothing deployed yet)", response: `{"next_action":"rollback"}`, want: 0},
+		{name: "wrong: proceed_to_backup (test never passed)", response: `{"next_action":"proceed_to_backup"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanMilestoneDateTest_Eval(t *testing.T) {
+	tc := planMilestoneDateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct number", response: "7", want: 1},
+		{name: "correct number with trailing reasoning line", response: "prep_data finishes on day 5, M starts then and takes 2 more days.\n7", want: 1},
+		{name: "correct number with unit suffix", response: "day 7", want: 1},
+		{name: "wrong: used prep_env instead of the slower prerequisite", response: "5", want: 0},
+		{name: "wrong: summed durations serially", response: "10", want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanMilestoneWant_HandDerived cross-checks planMilestoneWant against
+// an independently hand-computed value.
+func TestPlanMilestoneWant_HandDerived(t *testing.T) {
+	const handDerived = 7
+	if planMilestoneWant != handDerived {
+		t.Errorf("planMilestoneWant = %d, hand-derived = %d", planMilestoneWant, handDerived)
+	}
+}
+
+func TestPlanResourceConstrainedTest_Eval(t *testing.T) {
+	tc := planResourceConstrainedTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct order",
+			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model","smoke_test"]`,
+			want:     1,
+		},
+		{
+			name:     "correct order fenced",
+			response: "```json\n[\"provision_ct\",\"install_graphann\",\"configure_network\",\"deploy_reranker_model\",\"smoke_test\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "wrong: ignores priority tie-break at the first choice",
+			response: `["configure_network","provision_ct","install_graphann","deploy_reranker_model","smoke_test"]`,
+			want:     0,
+		},
+		{
+			name:     "wrong: runs two steps at once conceptually (deploy before configure_network)",
+			response: `["provision_ct","install_graphann","deploy_reranker_model","configure_network","smoke_test"]`,
+			want:     0,
+		},
+		{
+			name:     "missing a step",
+			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model"]`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanResourceConstrainedTest_UniqueOrder independently proves, by
+// brute-force enumeration over all 5! permutations filtered against the
+// priority rule stated in the prompt, that planResourceConstrainedWant is
+// the ONLY order consistent with both the dependency graph and the
+// priority tie-break rule.
+func TestPlanResourceConstrainedTest_UniqueOrder(t *testing.T) {
+	all := planEnumerateTopoOrders(planResourceConstrainedSteps)
+
+	var consistent [][]string
+	for _, order := range all {
+		if planIsPriorityConsistent(order, planResourceConstrainedSteps, planResourceConstrainedPriority) {
+			consistent = append(consistent, order)
+		}
+	}
+	if len(consistent) != 1 {
+		t.Fatalf("expected exactly 1 priority-consistent topological order, found %d: %v", len(consistent), consistent)
+	}
+	if !reflect.DeepEqual(consistent[0], planResourceConstrainedWant) {
+		t.Errorf("unique consistent order = %v, want %v", consistent[0], planResourceConstrainedWant)
+	}
+}
+
+func TestPlanCycleDetectTest_Eval(t *testing.T) {
+	tc := planCycleDetectTest()
+
+	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
+	// out of a denominator of 3" (computed, not hardcoded, to avoid a
+	// float-literal precision mismatch against the evaluator's own
+	// float64(matched)/float64(denom) division).
+	twoOfThree := float64(2) / float64(3)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "exact correct set", response: `["c","d","e"]`, want: 1},
+		{name: "exact correct set, different order", response: `["e","c","d"]`, want: 1},
+		{name: "exact correct set, fenced", response: "```json\n[\"c\",\"d\",\"e\"]\n```", want: 1},
+		{name: "includes an acyclic task", response: `["a","c","d","e"]`, want: 0.75},
+		{name: "wrong: only two of the three cyclic tasks", response: `["c","d"]`, want: twoOfThree},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanCycleDetectTest_NoValidOrder independently confirms
+// planCycleSteps genuinely contains a cycle (the full graph admits zero
+// valid topological orders under brute-force enumeration), and that
+// removing the c-depends-on-e edge specifically restores at least one
+// valid order - pinpointing that edge, and thus c/d/e, as the cycle.
+func TestPlanCycleDetectTest_NoValidOrder(t *testing.T) {
+	if got := planEnumerateTopoOrders(planCycleSteps); len(got) != 0 {
+		t.Fatalf("expected zero valid topological orders for a graph containing a cycle, found %d: %v", len(got), got)
+	}
+
+	acyclic := make([]planStep, len(planCycleSteps))
+	copy(acyclic, planCycleSteps)
+	for i, s := range acyclic {
+		if s.ID == "c" {
+			acyclic[i] = planStep{ID: "c", Deps: []string{"b"}} // drop c's dependency on e
+		}
+	}
+	if got := planEnumerateTopoOrders(acyclic); len(got) == 0 {
+		t.Fatal("removing the c-depends-on-e edge should leave at least one valid topological order, found none")
+	}
+}
+
+func TestPlanMinimalReplanTest_Eval(t *testing.T) {
+	tc := planMinimalReplanTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct order", response: `["build","test","deploy","verify","rollback"]`, want: 1},
+		{name: "correct order fenced", response: "```json\n[\"build\",\"test\",\"deploy\",\"verify\",\"rollback\"]\n```", want: 1},
+		{name: "wrong: backup left in", response: `["build","test","backup","deploy","verify","rollback"]`, want: 0},
+		{name: "wrong: rollback before verify", response: `["build","test","deploy","rollback","verify"]`, want: 0},
+		{name: "missing a step", response: `["build","test","deploy","verify"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanMinimalReplanWant_UniqueOrder confirms, by brute-force
+// enumeration, that planMinimalReplanWant is the only ordering of the 5
+// remaining steps consistent with the requirement change's constraints.
+func TestPlanMinimalReplanWant_UniqueOrder(t *testing.T) {
+	steps := []planStep{
+		{ID: "build"},
+		{ID: "test", Deps: []string{"build"}},
+		{ID: "deploy", Deps: []string{"test"}},
+		{ID: "verify", Deps: []string{"deploy"}},
+		{ID: "rollback", Deps: []string{"verify"}},
+	}
+	valid := planEnumerateTopoOrders(steps)
+	if len(valid) != 1 {
+		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
+	}
+	if !reflect.DeepEqual(valid[0], planMinimalReplanWant) {
+		t.Errorf("unique valid order = %v, want %v", valid[0], planMinimalReplanWant)
+	}
+}