mirror of
https://github.com/lukaszraczylo/llm-testbench.git
synced 2026-10-12 00:45:19 +00:00
fix(eval): bound exec evaluators and pin their environment
Limit concurrent compile and run subprocesses. Heavy load no longer pushes correct solutions past the timeout. Build Go code before it runs, so a cold compile does not use the run timeout. Cap captured output at 1 MiB. Set a wait delay so a child that holds a pipe cannot hang the run. Report a timeout as "timed out after Ns". Pin TZ, PYTHONHASHSEED, PYTHONUTF8 and GOTOOLCHAIN for repeatable output.
This commit is contained in:
@@ -3,7 +3,9 @@ package eval
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func requireToolchain(t *testing.T, name string) {
|
||||
@@ -184,3 +186,46 @@ func TestExecEvaluators_ToolchainMissingSkipsNotFails(t *testing.T) {
|
||||
t.Errorf("CRun with empty PATH: Skipped = false, want true (%+v)", cScore)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBounded_TimeoutIsReported(t *testing.T) {
|
||||
if _, err := exec.LookPath("sleep"); err != nil {
|
||||
t.Skip("sleep not available")
|
||||
}
|
||||
_, _, err := runBounded(context.Background(), 100*time.Millisecond, t.TempDir(), nil, "sleep", "5")
|
||||
if err == nil || !strings.Contains(err.Error(), "timed out after") {
|
||||
t.Errorf("err = %v, want a timed out error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBounded_OutputIsCapped(t *testing.T) {
|
||||
if _, err := exec.LookPath("sh"); err != nil {
|
||||
t.Skip("sh not available")
|
||||
}
|
||||
stdout, _, err := runBounded(context.Background(), 10*time.Second, t.TempDir(), nil, "sh", "-c", "yes | head -c 5000000")
|
||||
if err != nil {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
if len(stdout) != maxExecOutput {
|
||||
t.Errorf("captured %d bytes, want cap %d", len(stdout), maxExecOutput)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBounded_GrandchildHoldingPipeDoesNotHang(t *testing.T) {
|
||||
if _, err := exec.LookPath("sh"); err != nil {
|
||||
t.Skip("sh not available")
|
||||
}
|
||||
start := time.Now()
|
||||
_, _, _ = runBounded(context.Background(), 200*time.Millisecond, t.TempDir(), nil, "sh", "-c", "sleep 30 & sleep 30")
|
||||
if elapsed := time.Since(start); elapsed > 10*time.Second {
|
||||
t.Errorf("took %v, want return shortly after timeout + WaitDelay", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinimalExecEnv_PinsDeterminismVars(t *testing.T) {
|
||||
env := strings.Join(minimalExecEnv(t.TempDir()), "\n")
|
||||
for _, want := range []string{"TZ=UTC", "PYTHONHASHSEED=0", "GOTOOLCHAIN=local"} {
|
||||
if !strings.Contains(env, want) {
|
||||
t.Errorf("env missing %s", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user