Merge branch 'worktree-agent-af28b622ee23b5cef'

internal/tests/agents.go

@@ -1,106 +1,14 @@
 package tests
 
-import (
+import "github.com/lukaszraczylo/llm-testbench/internal/testkit"
-	"github.com/lukaszraczylo/llm-testbench/internal/eval"
-	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
-)
 
+// registerAgentsTests registers every agents-category test. Each
+// subcategory - tool-routing, planning, and delegation - has its own
+// register function and source file (agents_toolrouting.go,
+// agents_planning.go, agents_delegation.go) to keep any one file from
+// growing past a few hundred lines.
 func registerAgentsTests(r *testkit.Registry) {
-	r.Register(agentToolRoutingTest())
+	registerAgentToolRoutingTests(r)
-	r.Register(agentPlanOrderingTest())
+	registerAgentPlanningTests(r)
-}
+	registerAgentDelegationTests(r)
-
-// agentToolRoutingRoster is the inline tool roster for
-// agentToolRoutingTest.
-const agentToolRoutingRoster = `- search_web: Search the public web for current information not in your
-  training data.
-- read_file: Read the contents of a file from the local filesystem.
-- run_shell: Execute an arbitrary shell command on the local machine.
-- query_db: Run a read-only SQL query against the application's Postgres
-  database.
-- send_email: Send an email to a specified recipient.
-- none: No tool is needed; answer directly from what you already know.`
-
-// agentToolRoutingTest: map 4 mini-tasks to the correct tool from a 6-tool
-// roster.
-//
-// ground truth: task1 is arithmetic answerable without any tool (none);
-// task2 needs a row count from a database table (query_db); task3 asks for
-// current information that cannot be in training data (search_web); task4
-// asks to notify someone by email (send_email).
-func agentToolRoutingTest() testkit.Test {
-	prompt := `You have access to this roster of tools:
-
-` + agentToolRoutingRoster + `
-
-For each of these 4 tasks, decide which single tool (or "none") is the
-correct one to use:
-
-task1: "What is 15% of 240?"
-task2: "How many rows are in the 'orders' table?"
-task3: "What is today's top headline on a major news site?"
-task4: "Notify the on-call engineer at [email protected] that the deploy
-finished."
-
-Respond with only a JSON object mapping each task id to the tool name:
-{"task1":"...","task2":"...","task3":"...","task4":"..."}`
-
-	evaluator := eval.Mean(
-		eval.JSONField("task1", "none"),
-		eval.JSONField("task2", "query_db"),
-		eval.JSONField("task3", "search_web"),
-		eval.JSONField("task4", "send_email"),
-	)
-
-	return testkit.Test{
-		ID:          "agent-tool-routing",
-		Category:    "agents",
-		Subcategory: "tool-routing",
-		Description: "Map 4 mini-tasks to the correct tool from a 6-tool roster.",
-		Prompt:      prompt,
-		MaxTokens:   300,
-		Eval:        evaluator,
-	}
-}
-
-// agentPlanOrderingWant is the unique valid step order implied by the
-// dependency constraints in agentPlanOrderingTest's prompt.
-var agentPlanOrderingWant = []string{"build", "test", "backup", "deploy", "verify", "rollback"}
-
-// agentPlanOrderingTest: order deployment steps that have explicit
-// pairwise dependencies.
-//
-// ground truth: the prompt states build must precede test (need a built
-// artifact to test), backup must happen only after tests pass and only
-// before deploy, deploy must follow backup, verify must follow deploy, and
-// rollback is only meaningful after a deploy whose verify step failed. This
-// pins a single valid total order: build, test, backup, deploy, verify,
-// rollback.
-func agentPlanOrderingTest() testkit.Test {
-	prompt := `You are planning a deployment with these steps: build, test,
-backup, deploy, verify, rollback.
-
-Constraints:
-- You must build the artifact before you can test it.
-- You only take a backup of the current production state once tests have
-  passed - there is no point backing up before you know the new build is
-  viable.
-- Deploy only happens after the backup completes.
-- After deploy, you must verify the new deployment is healthy.
-- Rollback (restoring the pre-deploy backup) only makes sense after a
-  deploy has happened, and only if verify fails.
-
-Give the single correct step order as a JSON array of step ids, e.g.
-["build","test",...]. Include all 6 steps, listing rollback in its position
-in the sequence even though it is conditional on verify failing.`
-
-	return testkit.Test{
-		ID:          "agent-plan-ordering",
-		Category:    "agents",
-		Subcategory: "planning",
-		Description: "Order 6 deployment steps under explicit pairwise dependency constraints.",
-		Prompt:      prompt,
-		MaxTokens:   300,
-		Eval:        eval.JSONStringArrayEquals(agentPlanOrderingWant),
-	}
 }

internal/tests/agents_delegation.go

@@ -0,0 +1,425 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerAgentDelegationTests(r *testkit.Registry) {
+	r.Register(delegTaskToSpecialistTest())
+	r.Register(delegBuildThenVerifyTest())
+	r.Register(delegHandoffContextTest())
+	r.Register(delegVerifyVsTrustTest())
+	r.Register(delegBatchVsSeparateTest())
+	r.Register(delegMainThreadVsDelegateTest())
+	r.Register(delegReviewerIndependenceTest())
+	r.Register(delegEscalationToHumanTest())
+	r.Register(delegParallelDispatchSafetyTest())
+	r.Register(delegMinimalPrivilegeTest())
+}
+
+// delegRoster is the inline specialist-agent roster shared by every
+// delegation test. It mirrors real specialist agents used in this kind of
+// workflow: a builder, an independent reviewer, a test author, a security
+// auditor, an infra debugger, a docs author, a performance specialist, a
+// release specialist, an external-only researcher, and a coordinator.
+const delegRoster = `- code-writer: Implements new features or bug fixes by writing and editing
+  source code directly.
+- code-reviewer: Reviews a diff or PR for correctness, style, and risk.
+  Does not write code itself.
+- test-writer: Writes new automated tests (unit or integration) for
+  existing or new code.
+- security-reviewer: Audits code or config for security vulnerabilities
+  (injection, auth, secrets handling).
+- devops-debugger: Diagnoses failing deployments, CI pipelines, and
+  infrastructure incidents.
+- docs-writer: Writes and updates prose documentation (READMEs, guides,
+  design docs).
+- performance-engineer: Profiles and optimizes runtime performance and
+  resource usage.
+- release-manager: Manages version tagging, changelog generation, and
+  release publishing. Does not write or modify application code.
+- web-researcher: Searches the public web and summarizes external
+  information. Does not read or write anything in the repository.
+- orchestrator: Plans and sequences a multi-agent workflow, dispatching to
+  the other specialists. Does not do specialist work itself.`
+
+// delegTaskToSpecialistTest: map 4 tasks to the correct specialist from the
+// roster.
+//
+// ground truth: diagnosing a stuck Kubernetes deploy is an infrastructure
+// incident (devops-debugger, not code-writer, since nothing is known yet to
+// be a code bug). Writing new unit tests for an evaluator is exactly
+// test-writer's job, not code-writer's. Checking password hashing is a
+// security audit (security-reviewer), not code-reviewer's general
+// correctness/style/risk pass. Publishing a version with a generated
+// changelog is release-manager's job specifically because it excludes
+// writing code.
+func delegTaskToSpecialistTest() testkit.Test {
+	prompt := `You can delegate to this roster of specialist agents:
+
+` + delegRoster + `
+
+For each of these 4 tasks, decide which single specialist is the correct
+one to dispatch to:
+
+task1: "Investigate why the Kubernetes deployment of graphann-web is stuck
+in ImagePullBackOff."
+task2: "Write unit tests covering the new cosine-similarity evaluator's
+edge cases."
+task3: "Check whether the new /admin login endpoint properly hashes
+passwords with bcrypt before storing them."
+task4: "Publish v1.4.0 with a changelog generated from the merged commits
+since v1.3.0."
+
+Respond with only a JSON object mapping each task id to the specialist
+name: {"task1":"...","task2":"...","task3":"...","task4":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task1", "devops-debugger"),
+		eval.JSONField("task2", "test-writer"),
+		eval.JSONField("task3", "security-reviewer"),
+		eval.JSONField("task4", "release-manager"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-task-to-specialist",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Map 4 tasks to the correct specialist agent from a 10-agent roster.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegBuildThenVerifyTest: pick the two specialists, in order, for a
+// build-then-independently-verify workflow.
+//
+// ground truth: "implemented" is code-writer's job. "independently checked
+// for correctness and risk before merge" is code-reviewer's roster
+// description verbatim - not test-writer, which writes tests rather than
+// reviewing the implementation, and not the same code-writer again, which
+// would not be independent.
+func delegBuildThenVerifyTest() testkit.Test {
+	prompt := `You can delegate to this roster of specialist agents:
+
+` + delegRoster + `
+
+Task: "A new caching layer needs to be implemented, and then someone
+independent needs to check it is correct and low-risk before it merges."
+
+Which TWO specialists should handle this, in order? Respond with only a
+JSON array of exactly 2 specialist names, e.g. ["a","b"].`
+
+	return testkit.Test{
+		ID:          "agent-deleg-build-then-verify",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Pick the two specialists, in order, for a build-then-independently-verify workflow.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"code-writer", "code-reviewer"}),
+	}
+}
+
+// delegHandoffContextCandidates is the enumerated list of possible context
+// items for delegHandoffContextTest, mixing genuinely essential items with
+// plausible-sounding noise.
+const delegHandoffContextCandidates = `- file_paths_touched
+- failing_test_output
+- reproduction_steps
+- acceptance_criteria
+- unrelated_agent_chat_history
+- full_git_log_since_project_start`
+
+// delegHandoffContextWant is the subset of candidates essential to a
+// failing-test-bug handoff.
+//
+// ground truth: code-writer needs to know exactly which files are involved
+// (file_paths_touched), what actually failed (failing_test_output), how to
+// reproduce it (reproduction_steps), and what "fixed" means
+// (acceptance_criteria). Chat history from unrelated work and the entire
+// project's git log are not about this bug and would only add noise the
+// specialist has to filter back out.
+var delegHandoffContextWant = []string{
+	"file_paths_touched",
+	"failing_test_output",
+	"reproduction_steps",
+	"acceptance_criteria",
+}
+
+func delegHandoffContextTest() testkit.Test {
+	prompt := `You are handing off a failing-test bug to code-writer to fix.
+Here are candidate items you could include in the handoff:
+
+` + delegHandoffContextCandidates + `
+
+Which of these candidates are essential to include in the handoff? Exclude
+anything that is unrelated noise. Respond with only a JSON array of the
+essential candidate names.`
+
+	return testkit.Test{
+		ID:          "agent-deleg-handoff-context",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Select the essential context items for a bug handoff from an enumerated list mixing essentials and noise.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet(delegHandoffContextWant),
+	}
+}
+
+// delegVerifyVsTrustTest: decide, for two changes of very different risk,
+// whether the result should be independently verified or simply trusted.
+//
+// ground truth: a one-line comment typo has no behavioral effect and is
+// trivially reversible, so trusting it without a separate verification pass
+// is reasonable. A change to authentication token validation logic is
+// security-sensitive and easy to get subtly wrong, so it must be
+// independently verified before being trusted.
+func delegVerifyVsTrustTest() testkit.Test {
+	prompt := `For each of these two completed changes, should the result be
+independently verified before trusting it, or is it reasonable to trust it
+as-is?
+
+scenario_a: "A one-line typo fix inside a code comment. No executable code
+changed."
+scenario_b: "A change to the authentication token validation logic."
+
+Respond with only a JSON object:
+{"scenario_a":"trust"|"verify","scenario_b":"trust"|"verify"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario_a", "trust"),
+		eval.JSONField("scenario_b", "verify"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-verify-vs-trust",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Decide verify-vs-trust for a trivial comment typo versus a security-sensitive auth logic change.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegBatchVsSeparateTest: decide whether related failures should be
+// batched into one dispatch or split into separate ones.
+//
+// ground truth: three failures sharing one root cause in one function are
+// fixed by one change, so one batched dispatch is strictly better - three
+// separate dispatches would triplicate the same diagnosis. A flaky network
+// timeout and an unrelated real logic bug in a different module have
+// different root causes and different fixes in different files, so they
+// should be separate dispatches rather than conflated into one.
+func delegBatchVsSeparateTest() testkit.Test {
+	prompt := `For each of these two situations, should the fixes be
+batched into one dispatch, or split into separate dispatches?
+
+scenario_a: "Three failing tests, all caused by the exact same off-by-one
+bug in the same function."
+scenario_b: "Two failing tests: one is a flaky network timeout in the
+integration suite, the other is a real logic bug in a completely unrelated
+module."
+
+Respond with only a JSON object:
+{"scenario_a":"batch"|"separate","scenario_b":"batch"|"separate"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario_a", "batch"),
+		eval.JSONField("scenario_b", "separate"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-batch-vs-separate",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Decide batch-vs-separate dispatch for a single-root-cause failure set versus two unrelated failures.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegMainThreadVsDelegateTest: decide whether a quick edit should be
+// handled on the main thread or delegated to a subagent.
+//
+// ground truth: a single-character typo fix in a file already open needs
+// no extra context-gathering or specialist skill, so handling it directly
+// is faster and simpler than the overhead of dispatching a subagent. A
+// 6-file refactor plus new tests is large enough to benefit from a
+// dedicated agent's focused context and is exactly the kind of work
+// delegation exists for.
+func delegMainThreadVsDelegateTest() testkit.Test {
+	prompt := `For each of these two tasks, should you handle it directly on
+the main thread, or delegate it to a subagent?
+
+scenario_a: "Fix a single-character typo in a log message string, in the
+file you already have open."
+scenario_b: "Refactor the authentication module across 6 files and add
+corresponding tests."
+
+Respond with only a JSON object:
+{"scenario_a":"main-thread"|"delegate","scenario_b":"main-thread"|"delegate"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario_a", "main-thread"),
+		eval.JSONField("scenario_b", "delegate"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-main-thread-vs-delegate",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Decide main-thread-vs-delegate for a trivial one-file typo fix versus a 6-file refactor plus tests.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegReviewerIndependenceTest: the same agent that made a change must not
+// be the one that signs off on it.
+//
+// ground truth: code-writer just implemented the caching layer, so
+// code-writer reviewing its own work is not independent; code-reviewer is
+// the roster's dedicated independent-review specialist. In the second
+// case, security-reviewer directly fixed the issue it found, which makes
+// security-reviewer the author of that fix for this purpose - so a
+// separate independent pass (code-reviewer) must confirm it, not
+// security-reviewer confirming its own fix.
+func delegReviewerIndependenceTest() testkit.Test {
+	prompt := `You can delegate to this roster of specialist agents:
+
+` + delegRoster + `
+
+case1: "code-writer just implemented a new caching layer. Who should review
+it before merge?"
+case2: "security-reviewer found a vulnerability during an audit and fixed
+it directly. Who should confirm the fix is correct before merge?"
+
+In both cases, the same agent that made the change must not be the one that
+signs off on it. Respond with only a JSON object:
+{"case1":"...","case2":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("case1", "code-reviewer"),
+		eval.JSONField("case2", "code-reviewer"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-reviewer-independence",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Require an independent reviewer rather than letting an agent sign off on its own change, in two framings.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegEscalationToHumanTest: decide whether a change should escalate to a
+// human before proceeding, or can proceed autonomously.
+//
+// ground truth: deleting a production database table with no verified
+// backup is irreversible and high-risk with no safe rollback path, which is
+// exactly when a human must be asked first. Adding a new optional CLI flag
+// with a sensible default is reversible, low-risk, and unambiguous, so it
+// can proceed without waiting on a human.
+func delegEscalationToHumanTest() testkit.Test {
+	prompt := `For each of these two requested changes, should you escalate
+to a human before proceeding, or proceed without asking?
+
+scenario_a: "Delete a production database table. No backup has been
+verified to exist."
+scenario_b: "Add a new optional CLI flag with a sensible default value."
+
+Respond with only a JSON object:
+{"scenario_a":"escalate"|"proceed","scenario_b":"escalate"|"proceed"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario_a", "escalate"),
+		eval.JSONField("scenario_b", "proceed"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-escalation-to-human",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Decide escalate-vs-proceed for an irreversible high-risk database deletion versus a low-risk CLI flag addition.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegParallelDispatchSafetyTest: spot which of two planned parallel
+// dispatch pairs would conflict by touching the same file.
+//
+// ground truth: pair_1's two dispatches both list b.go, so running them in
+// parallel risks a write conflict on that shared file - unsafe. pair_2's
+// two dispatches touch entirely disjoint files (x.go and y.go), so nothing
+// they do can collide - safe.
+func delegParallelDispatchSafetyTest() testkit.Test {
+	prompt := `You are planning to dispatch two subagents in parallel.
+
+pair_1: dispatch_1 will edit files [a.go, b.go]; dispatch_2 will edit files
+[b.go, c.go].
+pair_2: dispatch_3 will edit files [x.go]; dispatch_4 will edit files
+[y.go].
+
+For each pair, is it safe to run the two dispatches in parallel, or do they
+conflict over a shared file? Respond with only a JSON object:
+{"pair_1":"safe"|"conflict","pair_2":"safe"|"conflict"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("pair_1", "conflict"),
+		eval.JSONField("pair_2", "safe"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-parallel-dispatch-safety",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Spot a shared-file conflict between one pair of planned parallel dispatches, versus a disjoint, safe pair.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// delegMinimalPrivilegeTest: pick the specialist with the narrowest access
+// that is still adequate, rather than a broader one that could also do the
+// job.
+//
+// ground truth: summarizing a competitor's public pricing page needs no
+// repository access at all - web-researcher is the roster's only agent
+// explicitly scoped to external information with no repo access, which is
+// the minimal-privilege fit; orchestrator or code-writer would carry
+// unnecessary repo-write privilege for a task that never touches the repo.
+// Tagging a release from already-merged commits, without writing or
+// modifying application code, is exactly release-manager's scope, which
+// explicitly excludes code changes - code-writer would carry unneeded
+// code-editing privilege for a task that must not touch code.
+func delegMinimalPrivilegeTest() testkit.Test {
+	prompt := `You can delegate to this roster of specialist agents:
+
+` + delegRoster + `
+
+scenario_a: "Look up and summarize a competitor's public pricing page.
+Nothing in the repository needs to change."
+scenario_b: "Generate a changelog and tag a release version from commits
+that are already merged. Do not write or modify any application code."
+
+For each scenario, pick the specialist with the narrowest access that is
+still adequate for the task. Respond with only a JSON object:
+{"scenario_a":"...","scenario_b":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("scenario_a", "web-researcher"),
+		eval.JSONField("scenario_b", "release-manager"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-deleg-minimal-privilege",
+		Category:    "agents",
+		Subcategory: "delegation",
+		Description: "Pick the narrowest-access adequate specialist for a repo-free lookup and a code-free release task.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}

internal/tests/agents_delegation_test.go

@@ -0,0 +1,286 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestDelegTaskToSpecialistTest_Eval(t *testing.T) {
+	tc := delegTaskToSpecialistTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task1":"devops-debugger","task2":"test-writer","task3":"security-reviewer","task4":"release-manager"}`,
+			want:     1,
+		},
+		{
+			name:     "all correct fenced with prose",
+			response: "Here is my mapping:\n```json\n{\"task1\":\"devops-debugger\",\"task2\":\"test-writer\",\"task3\":\"security-reviewer\",\"task4\":\"release-manager\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "one wrong: code-writer for infra incident",
+			response: `{"task1":"code-writer","task2":"test-writer","task3":"security-reviewer","task4":"release-manager"}`,
+			want:     0.75,
+		},
+		{
+			name:     "one wrong: code-reviewer for security audit",
+			response: `{"task1":"devops-debugger","task2":"test-writer","task3":"code-reviewer","task4":"release-manager"}`,
+			want:     0.75,
+		},
+		{
+			name:     "all wrong",
+			response: `{"task1":"orchestrator","task2":"code-writer","task3":"code-reviewer","task4":"code-writer"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegBuildThenVerifyTest_Eval(t *testing.T) {
+	tc := delegBuildThenVerifyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct order", response: `["code-writer","code-reviewer"]`, want: 1},
+		{name: "correct order fenced", response: "```json\n[\"code-writer\",\"code-reviewer\"]\n```", want: 1},
+		{name: "correct order different case", response: `["Code-Writer","Code-Reviewer"]`, want: 1},
+		{name: "reversed order", response: `["code-reviewer","code-writer"]`, want: 0},
+		{name: "wrong second agent", response: `["code-writer","test-writer"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegHandoffContextTest_Eval(t *testing.T) {
+	tc := delegHandoffContextTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "exact correct set",
+			response: `["file_paths_touched","failing_test_output","reproduction_steps","acceptance_criteria"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, different order",
+			response: `["acceptance_criteria","reproduction_steps","file_paths_touched","failing_test_output"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, fenced",
+			response: "```json\n[\"file_paths_touched\",\"failing_test_output\",\"reproduction_steps\",\"acceptance_criteria\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "includes noise item",
+			response: `["file_paths_touched","failing_test_output","reproduction_steps","acceptance_criteria","unrelated_agent_chat_history"]`,
+			want:     0.8,
+		},
+		{
+			name:     "missing an essential item",
+			response: `["file_paths_touched","failing_test_output","reproduction_steps"]`,
+			want:     0.75,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegVerifyVsTrustTest_Eval(t *testing.T) {
+	tc := delegVerifyVsTrustTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"scenario_a":"trust","scenario_b":"verify"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"trust\",\"scenario_b\":\"verify\"}\n```", want: 1},
+		{name: "scenario_a wrong", response: `{"scenario_a":"verify","scenario_b":"verify"}`, want: 0.5},
+		{name: "scenario_b wrong", response: `{"scenario_a":"trust","scenario_b":"trust"}`, want: 0.5},
+		{name: "both swapped", response: `{"scenario_a":"verify","scenario_b":"trust"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegBatchVsSeparateTest_Eval(t *testing.T) {
+	tc := delegBatchVsSeparateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"scenario_a":"batch","scenario_b":"separate"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"batch\",\"scenario_b\":\"separate\"}\n```", want: 1},
+		{name: "scenario_a wrong", response: `{"scenario_a":"separate","scenario_b":"separate"}`, want: 0.5},
+		{name: "scenario_b wrong", response: `{"scenario_a":"batch","scenario_b":"batch"}`, want: 0.5},
+		{name: "both swapped", response: `{"scenario_a":"separate","scenario_b":"batch"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegMainThreadVsDelegateTest_Eval(t *testing.T) {
+	tc := delegMainThreadVsDelegateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"scenario_a":"main-thread","scenario_b":"delegate"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"main-thread\",\"scenario_b\":\"delegate\"}\n```", want: 1},
+		{name: "scenario_a wrong", response: `{"scenario_a":"delegate","scenario_b":"delegate"}`, want: 0.5},
+		{name: "scenario_b wrong", response: `{"scenario_a":"main-thread","scenario_b":"main-thread"}`, want: 0.5},
+		{name: "both swapped", response: `{"scenario_a":"delegate","scenario_b":"main-thread"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegReviewerIndependenceTest_Eval(t *testing.T) {
+	tc := delegReviewerIndependenceTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"case1":"code-reviewer","case2":"code-reviewer"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"case1\":\"code-reviewer\",\"case2\":\"code-reviewer\"}\n```", want: 1},
+		{name: "case1 wrong: self-review", response: `{"case1":"code-writer","case2":"code-reviewer"}`, want: 0.5},
+		{name: "case2 wrong: self-sign-off", response: `{"case1":"code-reviewer","case2":"security-reviewer"}`, want: 0.5},
+		{name: "both wrong", response: `{"case1":"code-writer","case2":"security-reviewer"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegEscalationToHumanTest_Eval(t *testing.T) {
+	tc := delegEscalationToHumanTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"scenario_a":"escalate","scenario_b":"proceed"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"escalate\",\"scenario_b\":\"proceed\"}\n```", want: 1},
+		{name: "scenario_a wrong", response: `{"scenario_a":"proceed","scenario_b":"proceed"}`, want: 0.5},
+		{name: "scenario_b wrong", response: `{"scenario_a":"escalate","scenario_b":"escalate"}`, want: 0.5},
+		{name: "both swapped", response: `{"scenario_a":"proceed","scenario_b":"escalate"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegParallelDispatchSafetyTest_Eval(t *testing.T) {
+	tc := delegParallelDispatchSafetyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"pair_1":"conflict","pair_2":"safe"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"pair_1\":\"conflict\",\"pair_2\":\"safe\"}\n```", want: 1},
+		{name: "pair_1 wrong", response: `{"pair_1":"safe","pair_2":"safe"}`, want: 0.5},
+		{name: "pair_2 wrong", response: `{"pair_1":"conflict","pair_2":"conflict"}`, want: 0.5},
+		{name: "both swapped", response: `{"pair_1":"safe","pair_2":"conflict"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestDelegMinimalPrivilegeTest_Eval(t *testing.T) {
+	tc := delegMinimalPrivilegeTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"scenario_a":"web-researcher","scenario_b":"release-manager"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"scenario_a\":\"web-researcher\",\"scenario_b\":\"release-manager\"}\n```", want: 1},
+		{name: "scenario_a over-privileged", response: `{"scenario_a":"orchestrator","scenario_b":"release-manager"}`, want: 0.5},
+		{name: "scenario_b over-privileged", response: `{"scenario_a":"web-researcher","scenario_b":"code-writer"}`, want: 0.5},
+		{name: "both over-privileged", response: `{"scenario_a":"orchestrator","scenario_b":"code-writer"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}

internal/tests/agents_planning.go

@@ -0,0 +1,658 @@
+package tests
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerAgentPlanningTests(r *testkit.Registry) {
+	r.Register(agentPlanOrderingTest())
+	r.Register(planCriticalPathTest())
+	r.Register(planParallelStepsTest())
+	r.Register(planRollbackTriggerTest())
+	r.Register(planIdempotencyTest())
+	r.Register(planRepairTest())
+	r.Register(planMilestoneDateTest())
+	r.Register(planResourceConstrainedTest())
+	r.Register(planCycleDetectTest())
+	r.Register(planMinimalReplanTest())
+}
+
+// planStep is one node in a dependency graph shared by the planning
+// subcategory's ground-truth derivations: an id, the ids of steps that must
+// finish first, and (where relevant) a duration in the arbitrary time unit
+// the owning prompt states.
+type planStep struct {
+	ID       string
+	Deps     []string
+	Duration int
+}
+
+// planCriticalPathLength returns the length of the longest dependency chain
+// in steps, measured as the sum of Duration along that chain (the classic
+// critical-path-method "minimum time to finish everything, given unlimited
+// parallel capacity"). It assumes steps is acyclic; every planning fixture
+// in this file is verified acyclic (or, for planCycleSteps, deliberately
+// not) in agents_planning_test.go.
+func planCriticalPathLength(steps []planStep) int {
+	byID := make(map[string]planStep, len(steps))
+	for _, s := range steps {
+		byID[s.ID] = s
+	}
+	memo := make(map[string]int, len(steps))
+	var finish func(id string) int
+	finish = func(id string) int {
+		if v, ok := memo[id]; ok {
+			return v
+		}
+		s := byID[id]
+		start := 0
+		for _, dep := range s.Deps {
+			start = max(start, finish(dep))
+		}
+		v := start + s.Duration
+		memo[id] = v
+		return v
+	}
+	longest := 0
+	for _, s := range steps {
+		longest = max(longest, finish(s.ID))
+	}
+	return longest
+}
+
+// planWaveLayers groups steps into sequential stages, where a step's stage
+// is the earliest one possible after all of its Deps' stages have finished,
+// assuming unlimited parallel capacity (the standard "topological
+// generations" layering). Each stage is sorted alphabetically so the result
+// is a canonical value even though membership within a stage is a set.
+func planWaveLayers(steps []planStep) [][]string {
+	byID := make(map[string]planStep, len(steps))
+	for _, s := range steps {
+		byID[s.ID] = s
+	}
+	memo := make(map[string]int, len(steps))
+	var layerOf func(id string) int
+	layerOf = func(id string) int {
+		if v, ok := memo[id]; ok {
+			return v
+		}
+		s := byID[id]
+		layer := 0
+		for _, dep := range s.Deps {
+			layer = max(layer, layerOf(dep)+1)
+		}
+		memo[id] = layer
+		return layer
+	}
+
+	maxLayer := 0
+	byLayer := make(map[int][]string)
+	for _, s := range steps {
+		l := layerOf(s.ID)
+		byLayer[l] = append(byLayer[l], s.ID)
+		maxLayer = max(maxLayer, l)
+	}
+
+	out := make([][]string, maxLayer+1)
+	for l := 0; l <= maxLayer; l++ {
+		ids := byLayer[l]
+		sort.Strings(ids)
+		out[l] = ids
+	}
+	return out
+}
+
+// planScheduleSingleWorker simulates a single worker executing steps one at
+// a time: at every point it picks, among steps whose Deps are all already
+// done, the one with the lowest priority value (ties broken by the lower
+// ID), and runs it to completion before picking the next. It returns the
+// resulting total order.
+func planScheduleSingleWorker(steps []planStep, priority map[string]int) []string {
+	byID := make(map[string]planStep, len(steps))
+	for _, s := range steps {
+		byID[s.ID] = s
+	}
+	done := make(map[string]bool, len(steps))
+	order := make([]string, 0, len(steps))
+
+	for len(order) < len(steps) {
+		var ready []string
+		for _, s := range steps {
+			if done[s.ID] {
+				continue
+			}
+			allDepsDone := true
+			for _, dep := range s.Deps {
+				if !done[dep] {
+					allDepsDone = false
+					break
+				}
+			}
+			if allDepsDone {
+				ready = append(ready, s.ID)
+			}
+		}
+		sort.Slice(ready, func(i, j int) bool {
+			if priority[ready[i]] != priority[ready[j]] {
+				return priority[ready[i]] < priority[ready[j]]
+			}
+			return ready[i] < ready[j]
+		})
+		next := ready[0]
+		order = append(order, next)
+		done[next] = true
+	}
+	return order
+}
+
+// planWaveSetsEval scores full credit only when the response, parsed as a
+// JSON array of arrays of step ids, has the same number of stages as want
+// and each stage matches want's stage as an unordered, case-insensitive,
+// trimmed set. Stage order matters (a step cannot start until every step in
+// the prior stage has finished), but membership within a stage does not,
+// per the "canonicalize a set-valued answer" authoring rule - this is why a
+// bare JSONStringArrayEquals cannot be used here.
+type planWaveSetsEval struct {
+	want [][]string
+}
+
+func planWaveSets(want [][]string) eval.Evaluator {
+	return planWaveSetsEval{want: want}
+}
+
+func (p planWaveSetsEval) Evaluate(_ context.Context, response string) eval.Score {
+	raw, err := eval.ExtractJSON(response)
+	if err != nil {
+		return eval.Score{Value: 0, Detail: err.Error()}
+	}
+	var got [][]string
+	if err := json.Unmarshal([]byte(raw), &got); err != nil {
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("invalid JSON array of arrays: %v", err)}
+	}
+	if len(got) != len(p.want) {
+		return eval.Score{Value: 0, Detail: fmt.Sprintf("got %d stages, want %d", len(got), len(p.want))}
+	}
+	for i := range got {
+		if !planStringSetEqual(got[i], p.want[i]) {
+			return eval.Score{Value: 0, Detail: fmt.Sprintf("stage %d: got %v, want %v (compared as a set)", i, got[i], p.want[i])}
+		}
+	}
+	return eval.Score{Value: 1, Detail: "every stage matches as a set"}
+}
+
+// planStringSetEqual reports whether a and b contain the same strings,
+// ignoring order, case, and surrounding whitespace.
+func planStringSetEqual(a, b []string) bool {
+	norm := func(items []string) map[string]struct{} {
+		m := make(map[string]struct{}, len(items))
+		for _, s := range items {
+			m[strings.ToLower(strings.TrimSpace(s))] = struct{}{}
+		}
+		return m
+	}
+	am, bm := norm(a), norm(b)
+	if len(am) != len(bm) {
+		return false
+	}
+	for k := range am {
+		if _, ok := bm[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// agentPlanOrderingWant is the unique valid step order implied by the
+// dependency constraints in agentPlanOrderingTest's prompt.
+var agentPlanOrderingWant = []string{"build", "test", "backup", "deploy", "verify", "rollback"}
+
+// agentPlanOrderingTest: order deployment steps that have explicit
+// pairwise dependencies.
+//
+// ground truth: the prompt states build must precede test (need a built
+// artifact to test), backup must happen only after tests pass and only
+// before deploy, deploy must follow backup, verify must follow deploy, and
+// rollback is only meaningful after a deploy whose verify step failed. This
+// pins a single valid total order: build, test, backup, deploy, verify,
+// rollback.
+func agentPlanOrderingTest() testkit.Test {
+	prompt := `You are planning a deployment with these steps: build, test,
+backup, deploy, verify, rollback.
+
+Constraints:
+- You must build the artifact before you can test it.
+- You only take a backup of the current production state once tests have
+  passed - there is no point backing up before you know the new build is
+  viable.
+- Deploy only happens after the backup completes.
+- After deploy, you must verify the new deployment is healthy.
+- Rollback (restoring the pre-deploy backup) only makes sense after a
+  deploy has happened, and only if verify fails.
+
+Give the single correct step order as a JSON array of step ids, e.g.
+["build","test",...]. Include all 6 steps, listing rollback in its position
+in the sequence even though it is conditional on verify failing.`
+
+	return testkit.Test{
+		ID:          "agent-plan-ordering",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Order 6 deployment steps under explicit pairwise dependency constraints.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        eval.JSONStringArrayEquals(agentPlanOrderingWant),
+	}
+}
+
+// planCriticalPathSteps is a small CI/CD pipeline graph for
+// planCriticalPathTest: two branches (test / security_scan) fan out from
+// build and fan back in at package, while lint runs independently of both
+// and only gates deploy.
+var planCriticalPathSteps = []planStep{
+	{ID: "build", Duration: 3},
+	{ID: "lint", Duration: 2},
+	{ID: "test", Deps: []string{"build"}, Duration: 4},
+	{ID: "security_scan", Deps: []string{"build"}, Duration: 3},
+	{ID: "package", Deps: []string{"test", "security_scan"}, Duration: 1},
+	{ID: "deploy", Deps: []string{"package", "lint"}, Duration: 2},
+	{ID: "verify", Deps: []string{"deploy"}, Duration: 2},
+}
+
+// planCriticalPathWant is planCriticalPathLength(planCriticalPathSteps),
+// computed by running the algorithm rather than hardcoded.
+//
+// ground truth: the three source-to-sink chains are
+// build+test+package+deploy+verify = 3+4+1+2+2 = 12,
+// build+security_scan+package+deploy+verify = 3+3+1+2+2 = 11, and
+// lint+deploy+verify = 2+2+2 = 6. The longest (critical path) is 12.
+var planCriticalPathWant = planCriticalPathLength(planCriticalPathSteps)
+
+func planCriticalPathTest() testkit.Test {
+	prompt := `Here is a CI/CD pipeline as a set of steps with durations in
+minutes and dependencies:
+
+- build: 3 minutes, depends on nothing
+- lint: 2 minutes, depends on nothing
+- test: 4 minutes, depends on build
+- security_scan: 3 minutes, depends on build
+- package: 1 minute, depends on test AND security_scan
+- deploy: 2 minutes, depends on package AND lint
+- verify: 2 minutes, depends on deploy
+
+Assume unlimited parallel workers, so any steps whose dependencies are
+already satisfied can run at the same time. What is the critical path
+length in minutes - the minimum total time to finish every step, respecting
+all dependencies? Respond with only the number.`
+
+	return testkit.Test{
+		ID:          "agent-plan-critical-path",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Compute the critical path length in minutes of a 7-step CI/CD pipeline with two fan-out/fan-in branches.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], planCriticalPathWant, 0),
+	}
+}
+
+// planWaveSteps is a small 5-node graph for planParallelStepsTest: a and b
+// are independent sources, c depends only on a, d depends on both a and b,
+// and e depends on both c and d.
+var planWaveSteps = []planStep{
+	{ID: "a"},
+	{ID: "b"},
+	{ID: "c", Deps: []string{"a"}},
+	{ID: "d", Deps: []string{"a", "b"}},
+	{ID: "e", Deps: []string{"c", "d"}},
+}
+
+// planWaveWant is planWaveLayers(planWaveSteps), computed by running the
+// layering algorithm rather than hardcoded.
+//
+// ground truth: a and b have no dependencies, so both are stage 0. c
+// depends only on a (stage 0), so c is stage 1; d depends on a (stage 0)
+// and b (stage 0), so d is also stage 1. e depends on c and d, both stage
+// 1, so e is stage 2. Stages: [a,b], [c,d], [e].
+var planWaveWant = planWaveLayers(planWaveSteps)
+
+func planParallelStepsTest() testkit.Test {
+	prompt := `Here are 5 steps with dependencies:
+
+- a: depends on nothing
+- b: depends on nothing
+- c: depends on a
+- d: depends on a AND b
+- e: depends on c AND d
+
+Group these steps into sequential stages, where a step must be in the
+earliest stage possible once all of its dependencies' stages are complete,
+and assume unlimited parallel capacity (every step in a stage can run at
+the same time, since none of them depends on another step in the same
+stage). Respond with only a JSON array of arrays of step ids, ordered by
+stage, e.g. [["a","b"],["c"]]. Within a single stage's array, the order of
+ids does not matter.`
+
+	return testkit.Test{
+		ID:          "agent-plan-parallel-steps",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Group 5 dependency-graph steps into the sequential parallel-execution stages implied by the graph.",
+		Prompt:      prompt,
+		Eval:        planWaveSets(planWaveWant),
+	}
+}
+
+// planRollbackTriggerTest: pin exactly when a rollback decision should be
+// evaluated and what condition should trigger it, from an enumerated list
+// of options (forcing one deterministic answer rather than free text).
+//
+// ground truth: rollback restores the pre-deploy backup, so it is only ever
+// meaningful once a deploy has actually happened; evaluating it any earlier
+// (e.g. after test or backup) is premature since nothing has been deployed
+// yet to roll back from. The condition that should trigger it is
+// specifically a failed verify - a failed build or test should stop the
+// pipeline outright, long before deploy, and a manual request is not the
+// scenario described.
+func planRollbackTriggerTest() testkit.Test {
+	prompt := `A deployment pipeline runs these steps in order: build, test,
+backup, deploy, verify, rollback. Rollback restores the pre-deploy backup
+and is only ever useful once a deploy has actually happened.
+
+After which step should the pipeline evaluate whether to trigger rollback,
+and what specific condition should trigger it?
+
+Choose trigger_after from exactly one of: build, test, backup, deploy,
+verify.
+Choose condition from exactly one of: build_failure, test_failure,
+verify_failure, manual_request.
+
+Respond with only a JSON object:
+{"trigger_after":"...","condition":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("trigger_after", "verify"),
+		eval.JSONField("condition", "verify_failure"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-plan-rollback-trigger",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Pin the correct step and condition that should trigger a rollback, from enumerated option lists.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// planIdempotencySteps describes 5 pipeline steps with side-effect text
+// that determines whether rerunning each one from scratch is safe.
+const planIdempotencySteps = `- fetch_deps: downloads dependencies into a local cache directory,
+  overwriting any partial download from a prior attempt.
+- run_migrations: applies pending database schema migrations using a
+  migration tool that tracks which migrations already ran and skips them on
+  a rerun.
+- apply_terraform: runs "terraform apply", which reconciles infrastructure
+  to the declared state and is explicitly designed to be safe to rerun.
+- send_notification: posts a message to a Slack channel announcing the
+  deploy started, with no deduplication.
+- increment_deploy_counter: increments a deploy counter by 1 in a metrics
+  store, with no deduplication.`
+
+// planIdempotencyWant is the set of steps safe to retry from scratch.
+//
+// ground truth: fetch_deps overwrites cleanly, run_migrations explicitly
+// tracks and skips already-applied migrations, and apply_terraform
+// reconciles to a declared state - all three produce the same end result
+// whether run once or twice, so rerunning them after a crash is safe.
+// send_notification and increment_deploy_counter have no deduplication, so
+// rerunning either after a crash (rather than resuming past it) produces a
+// duplicate Slack message or an inflated counter - both unsafe to retry.
+var planIdempotencyWant = []string{"fetch_deps", "run_migrations", "apply_terraform"}
+
+func planIdempotencyTest() testkit.Test {
+	prompt := `Here are 5 pipeline steps and what each one does:
+
+` + planIdempotencySteps + `
+
+If the pipeline crashes partway through and you restart it from the
+beginning, which of these steps are safe to retry (idempotent - rerunning
+produces the same end result, with no bad side effect like a duplicate
+notification or an inflated counter)? Respond with only a JSON array of the
+safe-to-retry step ids.`
+
+	return testkit.Test{
+		ID:          "agent-plan-idempotency",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Identify which of 5 pipeline steps are safe to retry from scratch after a crash, from their stated side effects.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet(planIdempotencyWant),
+	}
+}
+
+// planRepairTest: a pipeline fails during "test" - before backup or deploy
+// have run - so the correct repair action is to fix the code and rerun
+// test, not to (mistakenly) treat the failure as something rollback or a
+// later step could address.
+//
+// ground truth: the original plan is build, test, backup, deploy, verify,
+// rollback (same as agent-plan-ordering). Test failed right after build
+// succeeded, so nothing has been backed up or deployed yet. rollback
+// requires an existing deploy to undo, and backup/deploy/verify all require
+// a passing test first - none of them is reachable or meaningful from this
+// failure point. The correct next action is to fix the code and rerun
+// test, restarting the sequence from there.
+func planRepairTest() testkit.Test {
+	prompt := `A deployment pipeline runs: build, test, backup, deploy,
+verify, rollback, in that order, with the same dependency rules as a
+standard pipeline (each step requires the previous ones to have succeeded;
+rollback requires an actual deploy to undo).
+
+On this run, build succeeded, then test failed. Nothing after test has run
+yet - no backup was taken and nothing was deployed.
+
+What is the correct next action? Choose exactly one from: proceed_to_backup,
+deploy, rollback, fix_and_rerun_test, abort_pipeline_permanently.
+
+Respond with only a JSON object: {"next_action":"..."}`
+
+	return testkit.Test{
+		ID:          "agent-plan-repair",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Pick the correct repair action after a pipeline fails at 'test', before anything was backed up or deployed.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("next_action", "fix_and_rerun_test"),
+	}
+}
+
+// planMilestoneWant is max(3, 5) + 2, computed via the same reasoning the
+// prompt describes rather than hardcoded.
+//
+// ground truth: prep_env finishes on day 0+3=3, prep_data finishes on day
+// 0+5=5. Milestone M cannot start until BOTH finish, so M starts on
+// max(3,5)=5, and finishes 2 days later, on day 5+2=7.
+var planMilestoneWant = max(3, 5) + 2
+
+func planMilestoneDateTest() testkit.Test {
+	prompt := `Two prerequisite work items both start on day 0 and run in
+parallel:
+- prep_env takes 3 days.
+- prep_data takes 5 days.
+
+Milestone M can only start once BOTH prep_env and prep_data have finished,
+and M itself takes 2 days once it starts.
+
+On what day number does milestone M finish (counting the day work starts
+as day 0, and each duration in whole days with no gaps)? Respond with only
+the number.`
+
+	return testkit.Test{
+		ID:          "agent-plan-milestone-date",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Compute the finish day of a milestone that depends on the slower of two parallel prerequisites plus its own duration.",
+		Prompt:      prompt,
+		Eval:        eval.Numeric(eval.ExtractLastNumber[int], planMilestoneWant, 0),
+	}
+}
+
+// planResourceConstrainedSteps models a single-engineer homelab deploy:
+// provisioning a Proxmox LXC container and installing software on it can
+// only happen after the container exists, while network configuration is
+// independent, and the final deploy step needs both branches done.
+var planResourceConstrainedSteps = []planStep{
+	{ID: "provision_ct"},
+	{ID: "configure_network"},
+	{ID: "install_graphann", Deps: []string{"provision_ct"}},
+	{ID: "deploy_reranker_model", Deps: []string{"configure_network", "install_graphann"}},
+	{ID: "smoke_test", Deps: []string{"deploy_reranker_model"}},
+}
+
+// planResourceConstrainedPriority is the "do the lower number first when
+// multiple steps are ready" tie-break rule stated in the prompt.
+var planResourceConstrainedPriority = map[string]int{
+	"provision_ct":          1,
+	"configure_network":     2,
+	"install_graphann":      1,
+	"deploy_reranker_model": 2,
+	"smoke_test":            1,
+}
+
+// planResourceConstrainedWant is
+// planScheduleSingleWorker(planResourceConstrainedSteps,
+// planResourceConstrainedPriority), computed by running the scheduler
+// rather than hardcoded. agents_planning_test.go independently proves this
+// is the ONLY order consistent with both the dependency graph and the
+// priority rule, by brute-force enumeration.
+//
+// ground truth: initially ready = {provision_ct(p1), configure_network(p2)}
+// -> pick provision_ct (lower priority). Then ready =
+// {configure_network(p2), install_graphann(p1)} -> pick install_graphann.
+// Then ready = {configure_network(p2)} only (deploy_reranker_model still
+// needs configure_network) -> pick configure_network. Then ready =
+// {deploy_reranker_model(p2)} -> pick it. Finally smoke_test.
+var planResourceConstrainedWant = planScheduleSingleWorker(planResourceConstrainedSteps, planResourceConstrainedPriority)
+
+func planResourceConstrainedTest() testkit.Test {
+	prompt := `Only one engineer is available, so steps must run strictly
+one at a time - never two at once, even if their dependencies allow it.
+
+Steps, dependencies, and priority (lower number = higher priority):
+- provision_ct: depends on nothing, priority 1
+- configure_network: depends on nothing, priority 2
+- install_graphann: depends on provision_ct, priority 1
+- deploy_reranker_model: depends on configure_network AND
+  install_graphann, priority 2
+- smoke_test: depends on deploy_reranker_model, priority 1
+
+Rule: whenever more than one step is ready to start (its dependencies are
+all done) at the same time, the engineer always does the lower-priority-
+number step first.
+
+Give the resulting single-file execution order as a JSON array of step ids.
+Respond with only the JSON array.`
+
+	return testkit.Test{
+		ID:          "agent-plan-resource-constrained",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Order 5 steps for a single available worker under both a dependency graph and a priority tie-break rule.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals(planResourceConstrainedWant),
+	}
+}
+
+// planCycleSteps has a genuine circular dependency among c, d, and e: c
+// depends on e, d depends on c, and e depends on d, so none of the three
+// can ever become ready. a and b are unaffected.
+var planCycleSteps = []planStep{
+	{ID: "a"},
+	{ID: "b", Deps: []string{"a"}},
+	{ID: "c", Deps: []string{"b", "e"}},
+	{ID: "d", Deps: []string{"c"}},
+	{ID: "e", Deps: []string{"d"}},
+}
+
+// planCycleWant is the set of task ids in the circular dependency.
+//
+// ground truth: a has no deps (fine). b depends only on a (fine). c
+// depends on b AND e; d depends on c; e depends on d. Following the chain
+// from c: c needs e, e needs d, d needs c - a genuine cycle among c, d, and
+// e, none of which can ever be satisfied. agents_planning_test.go
+// independently confirms this by brute-force enumeration: the full graph
+// admits zero valid topological orders, and removing just the c-depends-
+// on-e edge restores at least one.
+var planCycleWant = []string{"c", "d", "e"}
+
+func planCycleDetectTest() testkit.Test {
+	prompt := `Here are 5 tasks with dependencies:
+
+- a: depends on nothing
+- b: depends on a
+- c: depends on b AND e
+- d: depends on c
+- e: depends on d
+
+One of these dependency chains loops back on itself, so those tasks can
+never all be completed. Which task ids form that circular dependency?
+Respond with only a JSON array of the task ids in the cycle.`
+
+	return testkit.Test{
+		ID:          "agent-plan-cycle-detect",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Identify which 3 of 5 tasks form a circular dependency that can never be satisfied.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringSet(planCycleWant),
+	}
+}
+
+// planMinimalReplanWant is agent-plan-ordering's order with backup removed,
+// since the storage requirement change removes only that one step and its
+// downstream edge (deploy now follows test directly).
+var planMinimalReplanWant = []string{"build", "test", "deploy", "verify", "rollback"}
+
+// planMinimalReplanTest: a requirement change removes exactly one step
+// from the known-good plan; the correct replan keeps everything else
+// unchanged rather than re-deriving a new plan from scratch.
+//
+// ground truth: this restates agent-plan-ordering's original constraints
+// (build before test, deploy after test, verify after deploy, rollback
+// after a failed verify) minus the backup step, which the requirement
+// change states is no longer needed. The minimal update removes backup and
+// closes the gap it leaves - deploy now depends directly on test - and
+// otherwise preserves the original order.
+func planMinimalReplanTest() testkit.Test {
+	prompt := `The original deployment plan was: build, test, backup,
+deploy, verify, rollback, with these constraints:
+- build must precede test.
+- backup only happens after tests pass, and only before deploy.
+- deploy only happens after backup.
+- verify must follow deploy.
+- rollback only makes sense after a deploy whose verify step failed.
+
+Requirement change: production storage now uses continuous replication, so
+the backup step is no longer needed at all - remove it entirely, and deploy
+now proceeds directly once tests pass. Every other constraint above still
+applies unchanged.
+
+Give the minimally updated step order (5 steps: backup is dropped, nothing
+else changes) as a JSON array of step ids. Respond with only the JSON
+array.`
+
+	return testkit.Test{
+		ID:          "agent-plan-minimal-replan",
+		Category:    "agents",
+		Subcategory: "planning",
+		Description: "Update a known-good step order after a requirement change removes exactly one step, without re-deriving the rest.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals(planMinimalReplanWant),
+	}
+}

internal/tests/agents_planning_test.go

@@ -0,0 +1,537 @@
+package tests
+
+import (
+	"context"
+	"reflect"
+	"testing"
+)
+
+// planEnumerateTopoOrders returns every permutation of steps' ids that is a
+// valid topological order (every step appears after all of its Deps). It
+// exists only to prove, by exhaustive brute-force enumeration, that a
+// planning fixture admits exactly one valid order (or, for planCycleSteps,
+// admits none) - the small N (<=5) used by every fixture in this file
+// makes brute force cheap.
+func planEnumerateTopoOrders(steps []planStep) [][]string {
+	ids := make([]string, len(steps))
+	depsByID := make(map[string][]string, len(steps))
+	for i, s := range steps {
+		ids[i] = s.ID
+		depsByID[s.ID] = s.Deps
+	}
+
+	var permutations [][]string
+	var permute func(remaining, chosen []string)
+	permute = func(remaining, chosen []string) {
+		if len(remaining) == 0 {
+			order := make([]string, len(chosen))
+			copy(order, chosen)
+			permutations = append(permutations, order)
+			return
+		}
+		for i, id := range remaining {
+			rest := make([]string, 0, len(remaining)-1)
+			rest = append(rest, remaining[:i]...)
+			rest = append(rest, remaining[i+1:]...)
+			nextChosen := make([]string, len(chosen)+1)
+			copy(nextChosen, chosen)
+			nextChosen[len(chosen)] = id
+			permute(rest, nextChosen)
+		}
+	}
+	permute(ids, nil)
+
+	var valid [][]string
+	for _, order := range permutations {
+		position := make(map[string]int, len(order))
+		for i, id := range order {
+			position[id] = i
+		}
+		ok := true
+		for id, deps := range depsByID {
+			for _, dep := range deps {
+				if position[dep] > position[id] {
+					ok = false
+					break
+				}
+			}
+			if !ok {
+				break
+			}
+		}
+		if ok {
+			valid = append(valid, order)
+		}
+	}
+	return valid
+}
+
+// planIsPriorityConsistent independently re-derives, at every position in
+// order, the "ready" set (steps whose Deps are already placed earlier) and
+// checks the chosen step is the minimum by (priority, id) among them - the
+// same rule planResourceConstrainedTest's prompt states in words. It is
+// used to filter planEnumerateTopoOrders' output down to the orders
+// consistent with that rule, independently of planScheduleSingleWorker.
+func planIsPriorityConsistent(order []string, steps []planStep, priority map[string]int) bool {
+	depsByID := make(map[string][]string, len(steps))
+	for _, s := range steps {
+		depsByID[s.ID] = s.Deps
+	}
+	done := make(map[string]bool, len(steps))
+	for _, chosen := range order {
+		var ready []string
+		for _, s := range steps {
+			if done[s.ID] {
+				continue
+			}
+			allDone := true
+			for _, dep := range depsByID[s.ID] {
+				if !done[dep] {
+					allDone = false
+					break
+				}
+			}
+			if allDone {
+				ready = append(ready, s.ID)
+			}
+		}
+		best := ready[0]
+		for _, id := range ready[1:] {
+			if priority[id] < priority[best] || (priority[id] == priority[best] && id < best) {
+				best = id
+			}
+		}
+		if chosen != best {
+			return false
+		}
+		done[chosen] = true
+	}
+	return true
+}
+
+func TestAgentPlanOrderingTest_Eval(t *testing.T) {
+	tc := agentPlanOrderingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct order",
+			response: `["build","test","backup","deploy","verify","rollback"]`,
+			want:     1,
+		},
+		{
+			name:     "correct order fenced",
+			response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "backup before test violates dependency",
+			response: `["build","backup","test","deploy","verify","rollback"]`,
+			want:     0,
+		},
+		{
+			name:     "rollback before verify",
+			response: `["build","test","backup","deploy","rollback","verify"]`,
+			want:     0,
+		},
+		{
+			name:     "missing a step",
+			response: `["build","test","backup","deploy","verify"]`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestAgentPlanOrderingTest_UniqueOrder confirms, by brute-force
+// enumeration over all 6! permutations, that agentPlanOrderingWant is the
+// only ordering of the 6 steps consistent with the prompt's pairwise
+// constraints.
+func TestAgentPlanOrderingTest_UniqueOrder(t *testing.T) {
+	steps := []planStep{
+		{ID: "build"},
+		{ID: "test", Deps: []string{"build"}},
+		{ID: "backup", Deps: []string{"test"}},
+		{ID: "deploy", Deps: []string{"backup"}},
+		{ID: "verify", Deps: []string{"deploy"}},
+		{ID: "rollback", Deps: []string{"verify"}},
+	}
+	valid := planEnumerateTopoOrders(steps)
+	if len(valid) != 1 {
+		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
+	}
+	if !reflect.DeepEqual(valid[0], agentPlanOrderingWant) {
+		t.Errorf("unique valid order = %v, want %v", valid[0], agentPlanOrderingWant)
+	}
+}
+
+func TestPlanCriticalPathTest_Eval(t *testing.T) {
+	tc := planCriticalPathTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct number", response: "12", want: 1},
+		{name: "correct number with trailing reasoning line", response: "The critical path is build+test+package+deploy+verify.\n12", want: 1},
+		{name: "correct number with unit suffix", response: "12 minutes", want: 1},
+		{name: "wrong: shorter branch length", response: "11", want: 0},
+		{name: "wrong: sum of all durations ignoring parallelism", response: "17", want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanCriticalPathWant_HandDerived cross-checks planCriticalPathLength's
+// output against an independently hand-computed value (see the ground
+// truth comment on planCriticalPathWant), rather than only comparing the
+// algorithm to itself.
+func TestPlanCriticalPathWant_HandDerived(t *testing.T) {
+	const handDerived = 3 + 4 + 1 + 2 + 2 // build+test+package+deploy+verify
+	if planCriticalPathWant != handDerived {
+		t.Errorf("planCriticalPathWant = %d, hand-derived critical path = %d", planCriticalPathWant, handDerived)
+	}
+}
+
+func TestPlanParallelStepsTest_Eval(t *testing.T) {
+	tc := planParallelStepsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct stages", response: `[["a","b"],["c","d"],["e"]]`, want: 1},
+		{name: "correct stages, reordered within a stage", response: `[["b","a"],["d","c"],["e"]]`, want: 1},
+		{name: "correct stages, fenced", response: "```json\n[[\"a\",\"b\"],[\"c\",\"d\"],[\"e\"]]\n```", want: 1},
+		{name: "wrong: c and d not grouped together", response: `[["a","b"],["c"],["d"],["e"]]`, want: 0},
+		{name: "wrong: e placed too early", response: `[["a","b","e"],["c","d"]]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanWaveWant_HandDerived cross-checks planWaveLayers' output against
+// an independently hand-computed layering.
+func TestPlanWaveWant_HandDerived(t *testing.T) {
+	want := [][]string{{"a", "b"}, {"c", "d"}, {"e"}}
+	if !reflect.DeepEqual(planWaveWant, want) {
+		t.Errorf("planWaveWant = %v, hand-derived = %v", planWaveWant, want)
+	}
+}
+
+func TestPlanRollbackTriggerTest_Eval(t *testing.T) {
+	tc := planRollbackTriggerTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"trigger_after":"verify","condition":"verify_failure"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"trigger_after\":\"verify\",\"condition\":\"verify_failure\"}\n```", want: 1},
+		{name: "wrong trigger step", response: `{"trigger_after":"deploy","condition":"verify_failure"}`, want: 0.5},
+		{name: "wrong condition", response: `{"trigger_after":"verify","condition":"deploy_failure"}`, want: 0.5},
+		{name: "both wrong", response: `{"trigger_after":"test","condition":"manual_request"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanIdempotencyTest_Eval(t *testing.T) {
+	tc := planIdempotencyTest()
+
+	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
+	// out of a denominator of 3" (computed, not hardcoded, to avoid a
+	// float-literal precision mismatch against the evaluator's own
+	// float64(matched)/float64(denom) division).
+	twoOfThree := float64(2) / float64(3)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "exact correct set",
+			response: `["fetch_deps","run_migrations","apply_terraform"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, different order",
+			response: `["apply_terraform","fetch_deps","run_migrations"]`,
+			want:     1,
+		},
+		{
+			name:     "exact correct set, fenced",
+			response: "```json\n[\"fetch_deps\",\"run_migrations\",\"apply_terraform\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "includes an unsafe step",
+			response: `["fetch_deps","run_migrations","apply_terraform","send_notification"]`,
+			want:     0.75,
+		},
+		{
+			name:     "missing a safe step",
+			response: `["fetch_deps","run_migrations"]`,
+			want:     twoOfThree,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanRepairTest_Eval(t *testing.T) {
+	tc := planRepairTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: fix_and_rerun_test", response: `{"next_action":"fix_and_rerun_test"}`, want: 1},
+		{name: "correct, fenced with prose", response: "The right move:\n```json\n{\"next_action\":\"fix_and_rerun_test\"}\n```", want: 1},
+		{name: "correct, uppercase", response: `{"next_action":"FIX_AND_RERUN_TEST"}`, want: 1},
+		{name: "wrong: rollback (nothing deployed yet)", response: `{"next_action":"rollback"}`, want: 0},
+		{name: "wrong: proceed_to_backup (test never passed)", response: `{"next_action":"proceed_to_backup"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestPlanMilestoneDateTest_Eval(t *testing.T) {
+	tc := planMilestoneDateTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct number", response: "7", want: 1},
+		{name: "correct number with trailing reasoning line", response: "prep_data finishes on day 5, M starts then and takes 2 more days.\n7", want: 1},
+		{name: "correct number with unit suffix", response: "day 7", want: 1},
+		{name: "wrong: used prep_env instead of the slower prerequisite", response: "5", want: 0},
+		{name: "wrong: summed durations serially", response: "10", want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanMilestoneWant_HandDerived cross-checks planMilestoneWant against
+// an independently hand-computed value.
+func TestPlanMilestoneWant_HandDerived(t *testing.T) {
+	const handDerived = 7
+	if planMilestoneWant != handDerived {
+		t.Errorf("planMilestoneWant = %d, hand-derived = %d", planMilestoneWant, handDerived)
+	}
+}
+
+func TestPlanResourceConstrainedTest_Eval(t *testing.T) {
+	tc := planResourceConstrainedTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct order",
+			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model","smoke_test"]`,
+			want:     1,
+		},
+		{
+			name:     "correct order fenced",
+			response: "```json\n[\"provision_ct\",\"install_graphann\",\"configure_network\",\"deploy_reranker_model\",\"smoke_test\"]\n```",
+			want:     1,
+		},
+		{
+			name:     "wrong: ignores priority tie-break at the first choice",
+			response: `["configure_network","provision_ct","install_graphann","deploy_reranker_model","smoke_test"]`,
+			want:     0,
+		},
+		{
+			name:     "wrong: runs two steps at once conceptually (deploy before configure_network)",
+			response: `["provision_ct","install_graphann","deploy_reranker_model","configure_network","smoke_test"]`,
+			want:     0,
+		},
+		{
+			name:     "missing a step",
+			response: `["provision_ct","install_graphann","configure_network","deploy_reranker_model"]`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanResourceConstrainedTest_UniqueOrder independently proves, by
+// brute-force enumeration over all 5! permutations filtered against the
+// priority rule stated in the prompt, that planResourceConstrainedWant is
+// the ONLY order consistent with both the dependency graph and the
+// priority tie-break rule.
+func TestPlanResourceConstrainedTest_UniqueOrder(t *testing.T) {
+	all := planEnumerateTopoOrders(planResourceConstrainedSteps)
+
+	var consistent [][]string
+	for _, order := range all {
+		if planIsPriorityConsistent(order, planResourceConstrainedSteps, planResourceConstrainedPriority) {
+			consistent = append(consistent, order)
+		}
+	}
+	if len(consistent) != 1 {
+		t.Fatalf("expected exactly 1 priority-consistent topological order, found %d: %v", len(consistent), consistent)
+	}
+	if !reflect.DeepEqual(consistent[0], planResourceConstrainedWant) {
+		t.Errorf("unique consistent order = %v, want %v", consistent[0], planResourceConstrainedWant)
+	}
+}
+
+func TestPlanCycleDetectTest_Eval(t *testing.T) {
+	tc := planCycleDetectTest()
+
+	// twoOfThree is JSONStringSet's Jaccard-style overlap for "2 matched
+	// out of a denominator of 3" (computed, not hardcoded, to avoid a
+	// float-literal precision mismatch against the evaluator's own
+	// float64(matched)/float64(denom) division).
+	twoOfThree := float64(2) / float64(3)
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "exact correct set", response: `["c","d","e"]`, want: 1},
+		{name: "exact correct set, different order", response: `["e","c","d"]`, want: 1},
+		{name: "exact correct set, fenced", response: "```json\n[\"c\",\"d\",\"e\"]\n```", want: 1},
+		{name: "includes an acyclic task", response: `["a","c","d","e"]`, want: 0.75},
+		{name: "wrong: only two of the three cyclic tasks", response: `["c","d"]`, want: twoOfThree},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanCycleDetectTest_NoValidOrder independently confirms
+// planCycleSteps genuinely contains a cycle (the full graph admits zero
+// valid topological orders under brute-force enumeration), and that
+// removing the c-depends-on-e edge specifically restores at least one
+// valid order - pinpointing that edge, and thus c/d/e, as the cycle.
+func TestPlanCycleDetectTest_NoValidOrder(t *testing.T) {
+	if got := planEnumerateTopoOrders(planCycleSteps); len(got) != 0 {
+		t.Fatalf("expected zero valid topological orders for a graph containing a cycle, found %d: %v", len(got), got)
+	}
+
+	acyclic := make([]planStep, len(planCycleSteps))
+	copy(acyclic, planCycleSteps)
+	for i, s := range acyclic {
+		if s.ID == "c" {
+			acyclic[i] = planStep{ID: "c", Deps: []string{"b"}} // drop c's dependency on e
+		}
+	}
+	if got := planEnumerateTopoOrders(acyclic); len(got) == 0 {
+		t.Fatal("removing the c-depends-on-e edge should leave at least one valid topological order, found none")
+	}
+}
+
+func TestPlanMinimalReplanTest_Eval(t *testing.T) {
+	tc := planMinimalReplanTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct order", response: `["build","test","deploy","verify","rollback"]`, want: 1},
+		{name: "correct order fenced", response: "```json\n[\"build\",\"test\",\"deploy\",\"verify\",\"rollback\"]\n```", want: 1},
+		{name: "wrong: backup left in", response: `["build","test","backup","deploy","verify","rollback"]`, want: 0},
+		{name: "wrong: rollback before verify", response: `["build","test","deploy","rollback","verify"]`, want: 0},
+		{name: "missing a step", response: `["build","test","deploy","verify"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+// TestPlanMinimalReplanWant_UniqueOrder confirms, by brute-force
+// enumeration, that planMinimalReplanWant is the only ordering of the 5
+// remaining steps consistent with the requirement change's constraints.
+func TestPlanMinimalReplanWant_UniqueOrder(t *testing.T) {
+	steps := []planStep{
+		{ID: "build"},
+		{ID: "test", Deps: []string{"build"}},
+		{ID: "deploy", Deps: []string{"test"}},
+		{ID: "verify", Deps: []string{"deploy"}},
+		{ID: "rollback", Deps: []string{"verify"}},
+	}
+	valid := planEnumerateTopoOrders(steps)
+	if len(valid) != 1 {
+		t.Fatalf("expected exactly 1 valid topological order, found %d: %v", len(valid), valid)
+	}
+	if !reflect.DeepEqual(valid[0], planMinimalReplanWant) {
+		t.Errorf("unique valid order = %v, want %v", valid[0], planMinimalReplanWant)
+	}
+}

internal/tests/agents_test.go

@@ -1,84 +1,44 @@
 package tests
 
 import (
-	"context"
 	"testing"
+
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
 )
 
-func TestAgentToolRoutingTest_Eval(t *testing.T) {
+// TestRegisterAgentsTests_Wiring checks registerAgentsTests wires up all
-	tc := agentToolRoutingTest()
+// three agents subcategories - tool-routing, planning, delegation - with
+// 10 tests each, all categorized "agents", and no duplicate IDs across the
+// three sibling files (testkit.Registry.Register panics on a duplicate ID,
+// which would fail this test outright).
+func TestRegisterAgentsTests_Wiring(t *testing.T) {
+	r := testkit.NewRegistry()
+	registerAgentsTests(r)
 
-	tests := []struct {
+	const wantTotal = 30
-		name     string
+	if r.Len() != wantTotal {
-		response string
+		t.Fatalf("registerAgentsTests: registry has %d tests, want %d", r.Len(), wantTotal)
-		want     float64
-	}{
-		{
-			name:     "all correct",
-			response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
-			want:     1,
-		},
-		{
-			name:     "one wrong",
-			response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
-			want:     0.75,
-		},
-		{
-			name:     "all wrong",
-			response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
-			want:     0,
-		},
 	}
-	for _, tt := range tests {
+
-		t.Run(tt.name, func(t *testing.T) {
+	counts := map[string]int{}
-			got := tc.Eval.Evaluate(context.Background(), tt.response)
+	for _, tc := range r.All() {
-			if got.Value != tt.want {
+		if tc.Category != "agents" {
-				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			t.Errorf("test %q: Category = %q, want %q", tc.ID, tc.Category, "agents")
 		}
-		})
+		if tc.Prompt == "" {
+			t.Errorf("test %q: Prompt is empty", tc.ID)
 		}
-}
+		if tc.Eval == nil {
-
+			t.Errorf("test %q: Eval is nil", tc.ID)
-func TestAgentPlanOrderingTest_Eval(t *testing.T) {
-	tc := agentPlanOrderingTest()
-
-	tests := []struct {
-		name     string
-		response string
-		want     float64
-	}{
-		{
-			name:     "correct order",
-			response: `["build","test","backup","deploy","verify","rollback"]`,
-			want:     1,
-		},
-		{
-			name:     "correct order fenced",
-			response: "```json\n[\"build\",\"test\",\"backup\",\"deploy\",\"verify\",\"rollback\"]\n```",
-			want:     1,
-		},
-		{
-			name:     "backup before test violates dependency",
-			response: `["build","backup","test","deploy","verify","rollback"]`,
-			want:     0,
-		},
-		{
-			name:     "rollback before verify",
-			response: `["build","test","backup","deploy","rollback","verify"]`,
-			want:     0,
-		},
-		{
-			name:     "missing a step",
-			response: `["build","test","backup","deploy","verify"]`,
-			want:     0,
-		},
 		}
-	for _, tt := range tests {
+		if tc.Description == "" {
-		t.Run(tt.name, func(t *testing.T) {
+			t.Errorf("test %q: Description is empty", tc.ID)
-			got := tc.Eval.Evaluate(context.Background(), tt.response)
+		}
-			if got.Value != tt.want {
+		counts[tc.Subcategory]++
-				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+	}
+	for _, sub := range []string{"tool-routing", "planning", "delegation"} {
+		if counts[sub] != 10 {
+			t.Errorf("subcategory %q: got %d tests, want 10", sub, counts[sub])
 		}
-		})
 	}
 }

internal/tests/agents_toolrouting.go

@@ -0,0 +1,421 @@
+package tests
+
+import (
+	"github.com/lukaszraczylo/llm-testbench/internal/eval"
+	"github.com/lukaszraczylo/llm-testbench/internal/testkit"
+)
+
+func registerAgentToolRoutingTests(r *testkit.Registry) {
+	r.Register(agentToolRoutingTest())
+	r.Register(routeDistractorsTest())
+	r.Register(routeNoToolNeededTest())
+	r.Register(routeMultistepTest())
+	r.Register(routeCheapestToolTest())
+	r.Register(routeParallelDispatchTest())
+	r.Register(routeMissingParamTest())
+	r.Register(routeAmbiguousTest())
+	r.Register(routeChainingTest())
+	r.Register(routeSafetyTest())
+}
+
+// agentToolRoutingRoster is the inline tool roster for
+// agentToolRoutingTest, routeMultistepTest, routeParallelDispatchTest, and
+// routeChainingTest - all four scenarios are well served by this same
+// 6-tool roster, so it is shared rather than redefined per test.
+const agentToolRoutingRoster = `- search_web: Search the public web for current information not in your
+  training data.
+- read_file: Read the contents of a file from the local filesystem.
+- run_shell: Execute an arbitrary shell command on the local machine.
+- query_db: Run a read-only SQL query against the application's Postgres
+  database.
+- send_email: Send an email to a specified recipient.
+- none: No tool is needed; answer directly from what you already know.`
+
+// agentToolRoutingTest: map 4 mini-tasks to the correct tool from a 6-tool
+// roster.
+//
+// ground truth: task1 is arithmetic answerable without any tool (none);
+// task2 needs a row count from a database table (query_db); task3 asks for
+// current information that cannot be in training data (search_web); task4
+// asks to notify someone by email (send_email).
+func agentToolRoutingTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+For each of these 4 tasks, decide which single tool (or "none") is the
+correct one to use:
+
+task1: "What is 15% of 240?"
+task2: "How many rows are in the 'orders' table?"
+task3: "What is today's top headline on a major news site?"
+task4: "Notify the on-call engineer at [email protected] that the deploy
+finished."
+
+Respond with only a JSON object mapping each task id to the tool name:
+{"task1":"...","task2":"...","task3":"...","task4":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task1", "none"),
+		eval.JSONField("task2", "query_db"),
+		eval.JSONField("task3", "search_web"),
+		eval.JSONField("task4", "send_email"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Map 4 mini-tasks to the correct tool from a 6-tool roster.",
+		Prompt:      prompt,
+		MaxTokens:   300,
+		Eval:        evaluator,
+	}
+}
+
+// routeDistractorsRoster deliberately pairs each tool with a
+// similar-sounding decoy (fetch_url vs search_web, list_directory vs
+// read_file) so picking the right one requires reading the task detail, not
+// just keyword-matching "web" or "file".
+const routeDistractorsRoster = `- fetch_url: Retrieve the exact contents at a URL you already have; use
+  only when you already have the specific address.
+- search_web: Search the web to find pages or URLs matching a query, when
+  you do NOT already have a specific URL.
+- list_directory: List the file and directory names inside a local
+  directory path, without reading any file's contents.
+- read_file: Read the full contents of one specific local file, given its
+  path.
+- none: No tool is needed; answer directly from what you already know or
+  were already told.`
+
+// routeDistractorsTest: map 4 tasks to the correct tool from a roster where
+// every tool has a plausible, similarly-worded decoy.
+//
+// ground truth: task_a gives an exact URL, so fetch_url applies directly -
+// search_web is for when you lack a specific URL, which is not the case
+// here. task_b asks only for file NAMES inside a directory, not the
+// contents of any one file, so list_directory applies and read_file would
+// require picking a specific (unspecified) file. task_c's answer
+// (concurrency: 4) was already stated earlier in the same conversation, so
+// no tool is needed (none) - re-reading the file would be redundant.
+// task_d has no specific URL, only a topic to research, so search_web
+// applies and fetch_url cannot be used without a URL.
+func routeDistractorsTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + routeDistractorsRoster + `
+
+For each of these 4 tasks, decide which single tool (or "none") is the
+correct one to use:
+
+task_a: "Get the response body of exactly this URL:
+https://llm-gateway.example.com/v1/models."
+task_b: "You need the names of the files inside ./internal/tests/, not the
+contents of any of them."
+task_c: "Earlier in this conversation you already read config.yaml and it
+set concurrency: 4. What value did it set for concurrency?"
+task_d: "Find out which blog posts currently recommend goreleaser for Go
+release automation. You have no specific URL to start from."
+
+Respond with only a JSON object mapping each task id to the tool name:
+{"task_a":"...","task_b":"...","task_c":"...","task_d":"..."}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task_a", "fetch_url"),
+		eval.JSONField("task_b", "list_directory"),
+		eval.JSONField("task_c", "none"),
+		eval.JSONField("task_d", "search_web"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-distractors",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Map 4 tasks to the correct tool from a roster where every tool has a similarly-worded decoy.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// routeNoToolNeededTest: a single task whose answer is already fully
+// present in supplied context, so the correct tool choice is "none" even
+// though the roster tempts a read_file/search_web call.
+//
+// ground truth: the context sentence states the two dependencies verbatim.
+// Nothing needs to be fetched, read, or queried to answer the question
+// asked, so no tool is needed.
+func routeNoToolNeededTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+Context already given to you: "The llm-testbench Go module declares exactly
+two external dependencies in go.mod: golang.org/x/sync and gopkg.in/yaml.v3."
+
+Task: "Which two external dependencies does llm-testbench declare in
+go.mod?"
+
+Which single tool from the roster (or "none") is needed to answer this
+task, given the context already provided above? Respond with only a JSON
+object: {"tool":"..."}`
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-no-tool-needed",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Recognize that a task is already answerable from supplied context, so no tool is needed.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("tool", "none"),
+	}
+}
+
+// routeMultistepTest: a single task that genuinely requires two tools used
+// in a specific order, because the second call needs the first call's
+// result.
+//
+// ground truth: you cannot email a headline you have not yet found, so
+// search_web must run before send_email. There is no valid order that puts
+// send_email first, since its content depends on search_web's result.
+func routeMultistepTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+Task: "Find out today's top headline on a major news site, then email a
+one-line summary of it to [email protected]."
+
+This task requires more than one tool, used in a specific order because the
+second tool call needs the first tool call's result. List the required
+tools, in the order they must be called, as a JSON array, e.g.
+["tool_a","tool_b"]. Respond with only the JSON array.`
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-multistep",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Order the two tools a single task requires, where the second call depends on the first call's result.",
+		Prompt:      prompt,
+		Eval:        eval.JSONStringArrayEquals([]string{"search_web", "send_email"}),
+	}
+}
+
+// routeCostedRoster annotates each tool with an approximate relative cost,
+// for routeCheapestToolTest's "pick the cheapest adequate tool" decision.
+const routeCostedRoster = `- read_file (cost: free): Read a file already present on the local
+  filesystem.
+- query_db (cost: low): Run a read-only SQL query against a Postgres
+  replica.
+- search_web (cost: medium): Call an external web search API.
+- run_shell (cost: medium, requires elevated trust): Execute a shell
+  command on the local machine.
+- send_email (cost: low): Send an email to a specified recipient.
+- none (cost: free): No tool needed; answer directly.`
+
+// routeCheapestToolTest: pick the cheapest tool that is still adequate for
+// the task, from a roster where multiple tools could technically answer it.
+//
+// ground truth: the exact same value (concurrency: 4) is available from
+// three different sources - a local file, a mirrored DB table, and public
+// docs on the web - but read_file is free and requires no network or
+// elevated trust, while query_db and search_web cost more for an identical
+// result. The cheapest adequate tool is read_file.
+func routeCheapestToolTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + routeCostedRoster + `
+
+Task: "You need the current value of 'concurrency' from this project. It is
+set in the local file config.yaml, and the identical value is also mirrored
+in the Postgres table app_config, and also documented on the project's
+public website. Pick the cheapest tool that is still adequate to get this
+value."
+
+Respond with only a JSON object: {"tool":"..."}`
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-cheapest",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Pick the cheapest adequate tool when several roster tools could technically answer the same task.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("tool", "read_file"),
+	}
+}
+
+// routeParallelDispatchTest: decide, for two different tasks, whether the
+// tool calls they require should be dispatched in parallel or in sequence.
+//
+// ground truth: task_x's two queries are independent - neither's result
+// feeds the other - so they can run in parallel. task_y's second call
+// needs the exact count task_y's first call returns, so it must run
+// sequentially after the first completes.
+func routeParallelDispatchTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+task_x: "Query the row count of the 'orders' table, and separately query
+the row count of the 'shipments' table. Neither count depends on the
+other."
+task_y: "Query the row count of the 'orders' table, then email that exact
+count to [email protected]."
+
+For each task, should its required tool calls be dispatched in parallel or
+in sequence? Respond with only a JSON object:
+{"task_x":"parallel"|"sequential","task_y":"parallel"|"sequential"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task_x", "parallel"),
+		eval.JSONField("task_y", "sequential"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-parallel",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Decide parallel-vs-sequential tool dispatch for an independent-queries task and a data-dependent task.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// routeMissingParamRoster spells out send_email's required parameters, so a
+// task that omits one of them has a single correct answer: the tool cannot
+// be called as specified.
+const routeMissingParamRoster = `- send_email: Send an email. Required parameters: recipient (an email
+  address), subject, and body. All three must be known before this tool can
+  be called.
+- read_file: Read the contents of a local file, given its path.
+- none: No tool is needed; answer directly.`
+
+// routeMissingParamTest: a task instructs sending an email but never
+// supplies a recipient address, so the tool cannot be called as specified.
+//
+// ground truth: the roster states all three of send_email's parameters are
+// required before it can be called. The task supplies a subject/body-worthy
+// topic ("the deploy finished") but never names a recipient, so the correct
+// decision is "cannot" (call send_email), with the missing parameter being
+// recipient.
+func routeMissingParamTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + routeMissingParamRoster + `
+
+Task: "Send an email saying the deploy finished."
+
+Can send_email be called as specified? Respond with only a JSON object:
+{"decision":"can"|"cannot","missing_param":"<the missing required
+parameter, or empty string if none is missing>"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("decision", "cannot"),
+		eval.JSONField("missing_param", "recipient"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-missing-param",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Recognize a tool call is missing a required parameter and answer 'cannot' rather than guessing a value.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// routeAmbiguousTest: for two tasks, decide whether the correct move is to
+// ask a clarifying question before acting, or to act directly.
+//
+// ground truth: task_p names no specific files and describes a destructive,
+// irreversible action (deleting files), so multiple reasonable
+// interpretations exist and the safe move is to clarify before acting.
+// task_q names one exact path and is a safe, non-destructive read, so it
+// can be acted on directly.
+func routeAmbiguousTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+task_p: "Delete the old files."
+task_q: "Read the file located at ./config.yaml."
+
+For each task, is the correct next move to ask a clarifying question before
+doing anything, or to act directly using a tool? Respond with only a JSON
+object: {"task_p":"clarify"|"act","task_q":"clarify"|"act"}`
+
+	evaluator := eval.Mean(
+		eval.JSONField("task_p", "clarify"),
+		eval.JSONField("task_q", "act"),
+	)
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-ambiguous",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Decide clarify-vs-act for an ambiguous destructive task versus an unambiguous safe task.",
+		Prompt:      prompt,
+		Eval:        evaluator,
+	}
+}
+
+// routeChainingTest: given a tool's output as already-known context, pick
+// the next tool needed to act on it.
+//
+// ground truth: query_db already returned a list of email addresses; the
+// only roster tool that can notify each of them is send_email. run_shell
+// and search_web cannot send an email, and none is wrong because notifying
+// the users is still an outstanding, tool-requiring action.
+func routeChainingTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + agentToolRoutingRoster + `
+
+You already called query_db and it returned a list of 3 email addresses
+belonging to users with overdue invoices. Which single tool should you use
+next to notify each of them? Respond with only a JSON object:
+{"next_tool":"..."}`
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-chaining",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Chain from a completed tool call's output (a list of emails) to the correct next tool.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("next_tool", "send_email"),
+	}
+}
+
+// routeSafetyRoster distinguishes an explicitly read-only query tool from
+// an explicitly mutating one, for routeSafetyTest's least-privilege check.
+const routeSafetyRoster = `- query_db: Run a READ-ONLY SQL query (SELECT only) against the
+  application's Postgres database. Cannot modify any data.
+- modify_db: Execute an INSERT, UPDATE, or DELETE statement against the
+  application's Postgres database. Changes persisted data.
+- none: No tool is needed; answer directly.`
+
+// routeSafetyTest: the task only needs to report a count, never change any
+// data, so the read-only tool is correct even though a mutating tool could
+// technically also read a count first.
+//
+// ground truth: "just to report the count, not change anything" explicitly
+// rules out any need for modify_db. The least-privileged tool adequate for
+// a read-only reporting task is query_db.
+func routeSafetyTest() testkit.Test {
+	prompt := `You have access to this roster of tools:
+
+` + routeSafetyRoster + `
+
+Task: "Check how many orders are older than 30 days, just to report the
+count. Do not change anything."
+
+Which single tool is correct? Respond with only a JSON object:
+{"tool":"..."}`
+
+	return testkit.Test{
+		ID:          "agent-tool-routing-safety",
+		Category:    "agents",
+		Subcategory: "tool-routing",
+		Description: "Prefer the read-only tool over a mutating one when the task never requires changing data.",
+		Prompt:      prompt,
+		Eval:        eval.JSONField("tool", "query_db"),
+	}
+}

internal/tests/agents_toolrouting_test.go

@@ -0,0 +1,321 @@
+package tests
+
+import (
+	"context"
+	"testing"
+)
+
+func TestAgentToolRoutingTest_Eval(t *testing.T) {
+	tc := agentToolRoutingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task1":"none","task2":"query_db","task3":"search_web","task4":"send_email"}`,
+			want:     1,
+		},
+		{
+			name:     "all correct fenced with prose",
+			response: "Here is my answer:\n```json\n{\"task1\":\"none\",\"task2\":\"query_db\",\"task3\":\"search_web\",\"task4\":\"send_email\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "one wrong",
+			response: `{"task1":"run_shell","task2":"query_db","task3":"search_web","task4":"send_email"}`,
+			want:     0.75,
+		},
+		{
+			name:     "all wrong",
+			response: `{"task1":"run_shell","task2":"search_web","task3":"query_db","task4":"none"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteDistractorsTest_Eval(t *testing.T) {
+	tc := routeDistractorsTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task_a":"fetch_url","task_b":"list_directory","task_c":"none","task_d":"search_web"}`,
+			want:     1,
+		},
+		{
+			name:     "all correct with different casing and spacing",
+			response: "  {\"task_a\": \"Fetch_Url\", \"task_b\": \"List_Directory\", \"task_c\": \"None\", \"task_d\": \"Search_Web\"}  ",
+			want:     1,
+		},
+		{
+			name:     "distractor picked for task_a",
+			response: `{"task_a":"search_web","task_b":"list_directory","task_c":"none","task_d":"search_web"}`,
+			want:     0.75,
+		},
+		{
+			name:     "distractor picked for task_b",
+			response: `{"task_a":"fetch_url","task_b":"read_file","task_c":"none","task_d":"search_web"}`,
+			want:     0.75,
+		},
+		{
+			name:     "all wrong",
+			response: `{"task_a":"none","task_b":"fetch_url","task_c":"read_file","task_d":"list_directory"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteNoToolNeededTest_Eval(t *testing.T) {
+	tc := routeNoToolNeededTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: none", response: `{"tool":"none"}`, want: 1},
+		{name: "correct: none, fenced", response: "```json\n{\"tool\":\"none\"}\n```", want: 1},
+		{name: "correct: none, uppercase", response: `{"tool":"NONE"}`, want: 1},
+		{name: "wrong: read_file", response: `{"tool":"read_file"}`, want: 0},
+		{name: "wrong: search_web", response: `{"tool":"search_web"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteMultistepTest_Eval(t *testing.T) {
+	tc := routeMultistepTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct order", response: `["search_web","send_email"]`, want: 1},
+		{name: "correct order fenced", response: "```json\n[\"search_web\",\"send_email\"]\n```", want: 1},
+		{name: "correct order different case", response: `["Search_Web","Send_Email"]`, want: 1},
+		{name: "reversed order", response: `["send_email","search_web"]`, want: 0},
+		{name: "missing a tool", response: `["search_web"]`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteCheapestToolTest_Eval(t *testing.T) {
+	tc := routeCheapestToolTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: read_file", response: `{"tool":"read_file"}`, want: 1},
+		{name: "correct: read_file, fenced with prose", response: "The cheapest option is:\n```json\n{\"tool\":\"read_file\"}\n```", want: 1},
+		{name: "correct: read_file, uppercase", response: `{"tool":"READ_FILE"}`, want: 1},
+		{name: "wrong: query_db", response: `{"tool":"query_db"}`, want: 0},
+		{name: "wrong: search_web", response: `{"tool":"search_web"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteParallelDispatchTest_Eval(t *testing.T) {
+	tc := routeParallelDispatchTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "all correct",
+			response: `{"task_x":"parallel","task_y":"sequential"}`,
+			want:     1,
+		},
+		{
+			name:     "all correct fenced",
+			response: "```json\n{\"task_x\":\"parallel\",\"task_y\":\"sequential\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "task_x wrong",
+			response: `{"task_x":"sequential","task_y":"sequential"}`,
+			want:     0.5,
+		},
+		{
+			name:     "task_y wrong",
+			response: `{"task_x":"parallel","task_y":"parallel"}`,
+			want:     0.5,
+		},
+		{
+			name:     "both swapped",
+			response: `{"task_x":"sequential","task_y":"parallel"}`,
+			want:     0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteMissingParamTest_Eval(t *testing.T) {
+	tc := routeMissingParamTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{
+			name:     "correct: cannot, recipient",
+			response: `{"decision":"cannot","missing_param":"recipient"}`,
+			want:     1,
+		},
+		{
+			name:     "correct, fenced with prose",
+			response: "I cannot call send_email as specified.\n```json\n{\"decision\":\"cannot\",\"missing_param\":\"recipient\"}\n```",
+			want:     1,
+		},
+		{
+			name:     "correct, different case",
+			response: `{"decision":"CANNOT","missing_param":"Recipient"}`,
+			want:     1,
+		},
+		{
+			name:     "wrong decision",
+			response: `{"decision":"can","missing_param":"recipient"}`,
+			want:     0.5,
+		},
+		{
+			name:     "wrong missing param",
+			response: `{"decision":"cannot","missing_param":"subject"}`,
+			want:     0.5,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteAmbiguousTest_Eval(t *testing.T) {
+	tc := routeAmbiguousTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "all correct", response: `{"task_p":"clarify","task_q":"act"}`, want: 1},
+		{name: "all correct fenced", response: "```json\n{\"task_p\":\"clarify\",\"task_q\":\"act\"}\n```", want: 1},
+		{name: "task_p wrong", response: `{"task_p":"act","task_q":"act"}`, want: 0.5},
+		{name: "task_q wrong", response: `{"task_p":"clarify","task_q":"clarify"}`, want: 0.5},
+		{name: "both swapped", response: `{"task_p":"act","task_q":"clarify"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteChainingTest_Eval(t *testing.T) {
+	tc := routeChainingTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: send_email", response: `{"next_tool":"send_email"}`, want: 1},
+		{name: "correct, fenced with prose", response: "Next I should:\n```json\n{\"next_tool\":\"send_email\"}\n```", want: 1},
+		{name: "correct, uppercase", response: `{"next_tool":"SEND_EMAIL"}`, want: 1},
+		{name: "wrong: run_shell", response: `{"next_tool":"run_shell"}`, want: 0},
+		{name: "wrong: none", response: `{"next_tool":"none"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}
+
+func TestRouteSafetyTest_Eval(t *testing.T) {
+	tc := routeSafetyTest()
+
+	tests := []struct {
+		name     string
+		response string
+		want     float64
+	}{
+		{name: "correct: query_db", response: `{"tool":"query_db"}`, want: 1},
+		{name: "correct, fenced with prose", response: "The safest choice:\n```json\n{\"tool\":\"query_db\"}\n```", want: 1},
+		{name: "correct, uppercase", response: `{"tool":"QUERY_DB"}`, want: 1},
+		{name: "wrong: modify_db", response: `{"tool":"modify_db"}`, want: 0},
+		{name: "wrong: none", response: `{"tool":"none"}`, want: 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := tc.Eval.Evaluate(context.Background(), tt.response)
+			if got.Value != tt.want {
+				t.Errorf("Eval.Evaluate(%q) = %v, want %v (detail: %s)", tt.response, got.Value, tt.want, got.Detail)
+			}
+		})
+	}
+}