diff --git a/Cargo.lock b/Cargo.lock index b663733..ef19c18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1351,7 +1351,7 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "postil-cli" -version = "0.9.6" +version = "0.9.7" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index aa48dea..ae7c9db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "postil-cli" -version = "0.9.6" +version = "0.9.7" edition = "2024" description = "Postil: a low-noise AI review gate. Silent on clean PRs, hard gate on real risk." license = "Apache-2.0" diff --git a/bench/src/scorer-eval.test.ts b/bench/src/scorer-eval.test.ts index 5943490..837630a 100644 --- a/bench/src/scorer-eval.test.ts +++ b/bench/src/scorer-eval.test.ts @@ -1486,23 +1486,30 @@ describe("aggregate", () => { }); }); - test("fails a scorer with missing structured score fields", () => { + test("reports a timed-out scorer case without double-counting a structured failure", () => { const cases = qualificationCases(1); cases[0] = result({ id: TRUE_FINDING_CASES[0], timedOut: true, scorerModel: null, scorerConfidence: null, + reasonContractValid: false, }); + cases[1]!.passed = false; const aggregateResult = aggregate("scorer/model", cases, 1); expect(aggregateResult).toMatchObject({ timedOutCases: 1, - structuredFailures: 1, + structuredFailures: 0, + reasonContractFailures: 0, + trueFindingHighConfidence: TRUE_FINDING_CASES.length - 2, trueFindingCases: TRUE_FINDING_CASES.length - 1, admissionFailures: expect.arrayContaining(["1 case timeout(s)"]), passed: false, }); + expect(aggregateResult.admissionFailures).not.toEqual( + expect.arrayContaining([expect.stringContaining("structured-output failure")]), + ); expect(aggregateResult.admissionFailures).not.toEqual( expect.arrayContaining([expect.stringContaining("true risk(s)")]), ); diff --git a/bench/src/scorer-eval.ts b/bench/src/scorer-eval.ts index 3436f1f..d047e00 100644 --- a/bench/src/scorer-eval.ts +++ b/bench/src/scorer-eval.ts @@ -2228,7 +2228,9 @@ export function aggregate( repeats = DEFAULT_QUALIFICATION_REPEATS, ): ScorerEvalAggregate { const timedOutCases = cases.filter((c) => c.timedOut).length; - const structuredFailures = cases.filter((c) => isAdmissionFatalStructuralResult(c, model)).length; + const structuredFailures = cases.filter((c) => + !c.timedOut && isAdmissionFatalStructuralResult(c, model) + ).length; const trueCases = cases.filter((c) => c.scenario === "trueFinding"); const falseCases = cases.filter((c) => c.scenario === "falseFinding"); const eligibleTrueCases = trueCases.filter((c) => @@ -2245,7 +2247,9 @@ export function aggregate( const falseFindingDownscored = eligibleFalseCases.filter((c) => c.passed).length; const durations = cases.map((c) => c.durationMs).filter((value): value is number => value !== null); const costs = cases.map((c) => c.costUsd).filter((value): value is number => value !== null); - const reasonContractFailures = cases.filter((c) => !c.reasonContractValid).length; + const reasonContractFailures = cases.filter((c) => + !c.timedOut && !c.reasonContractValid + ).length; const p50DurationMs = percentile(durations, 0.5); const p95DurationMs = percentile(durations, 0.95); const maxDurationMs = durations.length > 0 ? Math.max(...durations) : 0; @@ -2261,7 +2265,7 @@ export function aggregate( } if (structuredFailures > 0) admissionFailures.push(`${structuredFailures} structured-output failure(s)`); if (timedOutCases > 0) admissionFailures.push(`${timedOutCases} case timeout(s)`); - const structuralPass = matrixComplete && structuredFailures === 0; + const structuralPass = matrixComplete && structuredFailures === 0 && timedOutCases === 0; if (structuralPass) { if (trueFindingHighConfidence !== eligibleTrueCases.length) { admissionFailures.push( diff --git a/tests/e2e.rs b/tests/e2e.rs index 95e6521..5741e09 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -9825,7 +9825,7 @@ async fn diff_fetch_failure_advisory_emits_envelope_and_exits_zero() { // The review check failed, while the gate stood aside with success. let reqs = server.received_requests().await.unwrap(); - let conclusions: Vec = reqs + let mut conclusions: Vec = reqs .iter() .filter(|r| r.method == wiremock::http::Method::PATCH) .map(|r| { @@ -9835,6 +9835,7 @@ async fn diff_fetch_failure_advisory_emits_envelope_and_exits_zero() { .to_string() }) .collect(); + conclusions.sort_unstable(); assert_eq!(conclusions, vec!["failure", "success"]); }