fix(eval): bound exec evaluators and pin their environment

Limit concurrent compile and run subprocesses. Heavy load no longer pushes correct solutions past the timeout.
Build Go code before it runs, so a cold compile does not use the run timeout.
Cap captured output at 1 MiB. Set a wait delay so a child that holds a pipe cannot hang the run.
Report a timeout as "timed out after Ns".
Pin TZ, PYTHONHASHSEED, PYTHONUTF8 and GOTOOLCHAIN for repeatable output.
This commit is contained in:
2026-10-02 01:01:00 +01:00
parent cfdb2c5aae
commit 6934d4df84
2 changed files with 136 additions and 53 deletions
+45
View File
@@ -3,7 +3,9 @@ package eval
import (
"context"
"os/exec"
"strings"
"testing"
"time"
)
func requireToolchain(t *testing.T, name string) {
@@ -184,3 +186,46 @@ func TestExecEvaluators_ToolchainMissingSkipsNotFails(t *testing.T) {
t.Errorf("CRun with empty PATH: Skipped = false, want true (%+v)", cScore)
}
}
func TestRunBounded_TimeoutIsReported(t *testing.T) {
if _, err := exec.LookPath("sleep"); err != nil {
t.Skip("sleep not available")
}
_, _, err := runBounded(context.Background(), 100*time.Millisecond, t.TempDir(), nil, "sleep", "5")
if err == nil || !strings.Contains(err.Error(), "timed out after") {
t.Errorf("err = %v, want a timed out error", err)
}
}
func TestRunBounded_OutputIsCapped(t *testing.T) {
if _, err := exec.LookPath("sh"); err != nil {
t.Skip("sh not available")
}
stdout, _, err := runBounded(context.Background(), 10*time.Second, t.TempDir(), nil, "sh", "-c", "yes | head -c 5000000")
if err != nil {
t.Fatalf("err = %v", err)
}
if len(stdout) != maxExecOutput {
t.Errorf("captured %d bytes, want cap %d", len(stdout), maxExecOutput)
}
}
func TestRunBounded_GrandchildHoldingPipeDoesNotHang(t *testing.T) {
if _, err := exec.LookPath("sh"); err != nil {
t.Skip("sh not available")
}
start := time.Now()
_, _, _ = runBounded(context.Background(), 200*time.Millisecond, t.TempDir(), nil, "sh", "-c", "sleep 30 & sleep 30")
if elapsed := time.Since(start); elapsed > 10*time.Second {
t.Errorf("took %v, want return shortly after timeout + WaitDelay", elapsed)
}
}
func TestMinimalExecEnv_PinsDeterminismVars(t *testing.T) {
env := strings.Join(minimalExecEnv(t.TempDir()), "\n")
for _, want := range []string{"TZ=UTC", "PYTHONHASHSEED=0", "GOTOOLCHAIN=local"} {
if !strings.Contains(env, want) {
t.Errorf("env missing %s", want)
}
}
}