diff --git a/internal/codex/benchmark.go b/internal/codex/benchmark.go index ac4dace..5736049 100644 --- a/internal/codex/benchmark.go +++ b/internal/codex/benchmark.go @@ -817,8 +817,6 @@ func (s *appServerSession) runBenchmark( } publish() turnTimeout := s.benchmarkTurnTimeout() - turnCtx, cancel := context.WithTimeout(ctx, turnTimeout) - defer cancel() temporary, err := os.MkdirTemp("", "codexometer-benchmark-") if err != nil { @@ -839,18 +837,21 @@ func (s *appServerSession) runBenchmark( if s.experimentalRawEvents { threadParams["experimentalRawEvents"] = true } - threadResult, err := s.call(turnCtx, "thread/start", threadParams, nil) + setupCtx, cancelSetup := context.WithTimeout(ctx, turnTimeout) + threadResult, err := s.call(setupCtx, "thread/start", threadParams, nil) if err != nil && s.experimentalRawEvents && experimentalAPIUnsupported(err) { // Older Codex versions may support the stable benchmark API but not raw // response telemetry. Retry this thread without the experimental field and // keep cumulative usage as the compatibility path for the rest of the suite. s.experimentalRawEvents = false delete(threadParams, "experimentalRawEvents") - threadResult, err = s.call(turnCtx, "thread/start", threadParams, nil) + threadResult, err = s.call(setupCtx, "thread/start", threadParams, nil) } + setupErr := fatalBenchmarkError(setupCtx, err) + cancelSetup() if err != nil { result.Failure = fmt.Sprintf("start thread: %v", err) - return result, fatalBenchmarkError(turnCtx, err) + return result, setupErr } var startedThread struct { Thread struct { @@ -876,6 +877,8 @@ func (s *appServerSession) runBenchmark( if combination.effort != "default" && combination.effort != "" { turnParams["effort"] = combination.effort } + turnCtx, cancelTurn := context.WithTimeout(ctx, turnTimeout) + defer cancelTurn() turnResponse, err := s.call(turnCtx, "turn/start", turnParams, nil) if err != nil { result.Duration = time.Since(startedAt) diff --git a/internal/codex/benchmark_test.go b/internal/codex/benchmark_test.go index 2fae3c6..bdf23b2 100644 --- a/internal/codex/benchmark_test.go +++ b/internal/codex/benchmark_test.go @@ -273,6 +273,35 @@ func TestRunBenchmarkConsumesStreamedResultAndUsage(t *testing.T) { } } +func TestRunBenchmarkRenewsTimeoutAfterThreadSetup(t *testing.T) { + message := string(rawJSON(map[string]string{"code": correctStarlarkSubmission})) + server, _ := newFakeBenchmarkServer() + server.turnTimeout = 200 * time.Millisecond + go func() { + time.Sleep(120 * time.Millisecond) + server.envelopes <- benchmarkEnvelope{ID: rawJSON(1), Result: rawJSON(map[string]any{ + "thread": map[string]string{"id": "thread-1"}, "model": "gpt-5.3-codex", + })} + time.Sleep(120 * time.Millisecond) + server.envelopes <- benchmarkEnvelope{ID: rawJSON(2), Result: rawJSON(map[string]any{ + "turn": map[string]string{"id": "turn-1"}, + })} + server.envelopes <- benchmarkEnvelope{Method: "item/completed", Params: rawJSON(map[string]any{ + "threadId": "thread-1", "turnId": "turn-1", "item": map[string]string{"type": "agentMessage", "text": message}, + })} + server.envelopes <- benchmarkEnvelope{Method: "turn/completed", Params: rawJSON(map[string]any{ + "threadId": "thread-1", "turn": map[string]any{"id": "turn-1", "status": "completed"}, + })} + }() + + result, fatalErr := server.runBenchmark(context.Background(), benchmarkCombination{ + model: benchmarkModel{Model: "gpt-5.3-codex", DisplayName: "GPT-5.3 Codex"}, effort: "high", + }, benchmarkTaskDefinitions[0]) + if fatalErr != nil || !result.Correct { + t.Fatalf("separate setup and turn deadlines failed: result=%#v fatal=%v", result, fatalErr) + } +} + func TestBenchmarkInteractionCaptureIsBoundedAndValidUTF8(t *testing.T) { countBounded := BenchmarkResult{} for index := 0; index < benchmarkInteractionCount+4; index++ { diff --git a/internal/version/VERSION b/internal/version/VERSION index d9df1bb..af88ba8 100644 --- a/internal/version/VERSION +++ b/internal/version/VERSION @@ -1 +1 @@ -0.11.0 +0.11.1