package eval import ( "context" "os/exec" "strings" "testing" "time" ) func requireToolchain(t *testing.T, name string) { t.Helper() if _, err := exec.LookPath(name); err != nil { t.Skipf("toolchain %s not found on PATH, skipping", name) } } // goHarness is a complete, standalone harness.go: its own package clause // and imports, calling Double (defined separately in solution.go, same // package) - no {{CODE}} placeholder, per GoRun's two-file contract. const goHarness = `package main import "fmt" func main() { fmt.Println(Double(21)) } ` func TestGoRun_Pass(t *testing.T) { requireToolchain(t, "go") e := GoRun(goHarness, "42") response := "```go\nfunc Double(n int) int { return n * 2 }\n```" got := e.Evaluate(context.Background(), response) if got.Value != 1 { t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail) } } func TestGoRun_WrongOutput(t *testing.T) { requireToolchain(t, "go") e := GoRun(goHarness, "42") response := "```go\nfunc Double(n int) int { return n + 2 }\n```" got := e.Evaluate(context.Background(), response) if got.Value != 0 { t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail) } } func TestGoRun_CompileError(t *testing.T) { requireToolchain(t, "go") e := GoRun(goHarness, "42") response := "```go\nfunc Double(n int) int { this is not valid go\n```" got := e.Evaluate(context.Background(), response) if got.Value != 0 { t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value) } if got.Skipped { t.Error("Evaluate() should not be Skipped for a compile error") } } func TestStripLeadingPackageClause(t *testing.T) { tests := []struct { name string code string want string }{ { name: "leading package clause is stripped", code: "package main\n\nfunc F() {}", want: "\nfunc F() {}", }, { name: "blank lines before package clause are tolerated", code: "\n\npackage foo\nfunc F() {}", want: "\n\nfunc F() {}", }, { name: "no package clause leaves code untouched", code: "import \"fmt\"\n\nfunc F() {}", want: "import \"fmt\"\n\nfunc F() {}", }, { name: "package clause not on the first line is left alone", code: "func F() {}\n\npackage main", want: "func F() {}\n\npackage main", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := stripLeadingPackageClause(tt.code) if got != tt.want { t.Errorf("stripLeadingPackageClause(%q) = %q, want %q", tt.code, got, tt.want) } }) } } func TestCheckToolchain_MissingBinaryIsSkipped(t *testing.T) { s := checkToolchain("definitely-not-a-real-toolchain-xyz") if s == nil || !s.Skipped { t.Fatal("checkToolchain() for missing binary should return Skipped score") } } const pyPassthroughHarness = PassthroughHarness func TestPyRun_Pass(t *testing.T) { requireToolchain(t, "python3") e := PyRun(pyPassthroughHarness, "OOMKilled=2\nImagePullBackOff=1") response := "```python\nprint('OOMKilled=2')\nprint('ImagePullBackOff=1')\n```" got := e.Evaluate(context.Background(), response) if got.Value != 1 { t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail) } } func TestPyRun_WrongOutput(t *testing.T) { requireToolchain(t, "python3") e := PyRun(pyPassthroughHarness, "OOMKilled=2") response := "```python\nprint('OOMKilled=3')\n```" got := e.Evaluate(context.Background(), response) if got.Value != 0 { t.Errorf("Evaluate() = %v, want 0 (detail: %s)", got.Value, got.Detail) } } func TestPyRun_RuntimeError(t *testing.T) { requireToolchain(t, "python3") e := PyRun(pyPassthroughHarness, "anything") response := "```python\nraise ValueError('boom')\n```" got := e.Evaluate(context.Background(), response) if got.Value != 0 || got.Skipped { t.Errorf("Evaluate() = %+v, want zero non-skipped score for runtime error", got) } } const cHarness = `#include {{CODE}} int main(void) { printf("%d\n", triple(7)); return 0; } ` func TestCRun_Pass(t *testing.T) { requireToolchain(t, "cc") e := CRun(cHarness, "21") response := "```c\nint triple(int n) { return n * 3; }\n```" got := e.Evaluate(context.Background(), response) if got.Value != 1 { t.Errorf("Evaluate() = %v, want 1 (detail: %s)", got.Value, got.Detail) } } func TestCRun_CompileError(t *testing.T) { requireToolchain(t, "cc") e := CRun(cHarness, "21") response := "```c\nthis is not valid c code {{{\n```" got := e.Evaluate(context.Background(), response) if got.Value != 0 { t.Errorf("Evaluate() = %v, want 0 for compile error", got.Value) } } func TestExecEvaluators_ToolchainMissingSkipsNotFails(t *testing.T) { // This test does not require any toolchain; it verifies the Skipped // contract directly against a bogus PATH so it runs in every environment. t.Setenv("PATH", t.TempDir()) goScore := GoRun(goHarness, "42").Evaluate(context.Background(), "```go\nfunc Double(n int) int { return n*2 }\n```") if !goScore.Skipped { t.Errorf("GoRun with empty PATH: Skipped = false, want true (%+v)", goScore) } pyScore := PyRun(pyPassthroughHarness, "x").Evaluate(context.Background(), "```python\nprint('x')\n```") if !pyScore.Skipped { t.Errorf("PyRun with empty PATH: Skipped = false, want true (%+v)", pyScore) } cScore := CRun(cHarness, "21").Evaluate(context.Background(), "```c\nint triple(int n){return n*3;}\n```") if !cScore.Skipped { t.Errorf("CRun with empty PATH: Skipped = false, want true (%+v)", cScore) } } func TestRunBounded_TimeoutIsReported(t *testing.T) { if _, err := exec.LookPath("sleep"); err != nil { t.Skip("sleep not available") } _, _, err := runBounded(context.Background(), 100*time.Millisecond, t.TempDir(), nil, "sleep", "5") if err == nil || !strings.Contains(err.Error(), "timed out after") { t.Errorf("err = %v, want a timed out error", err) } } func TestRunBounded_OutputIsCapped(t *testing.T) { if _, err := exec.LookPath("sh"); err != nil { t.Skip("sh not available") } stdout, _, err := runBounded(context.Background(), 10*time.Second, t.TempDir(), nil, "sh", "-c", "yes | head -c 5000000") if err != nil { t.Fatalf("err = %v", err) } if len(stdout) != maxExecOutput { t.Errorf("captured %d bytes, want cap %d", len(stdout), maxExecOutput) } } func TestRunBounded_GrandchildHoldingPipeDoesNotHang(t *testing.T) { if _, err := exec.LookPath("sh"); err != nil { t.Skip("sh not available") } start := time.Now() _, _, _ = runBounded(context.Background(), 200*time.Millisecond, t.TempDir(), nil, "sh", "-c", "sleep 30 & sleep 30") if elapsed := time.Since(start); elapsed > 10*time.Second { t.Errorf("took %v, want return shortly after timeout + WaitDelay", elapsed) } } func TestMinimalExecEnv_PinsDeterminismVars(t *testing.T) { env := strings.Join(minimalExecEnv(t.TempDir()), "\n") for _, want := range []string{"TZ=UTC", "PYTHONHASHSEED=0", "GOTOOLCHAIN=local"} { if !strings.Contains(env, want) { t.Errorf("env missing %s", want) } } }