From c819758c352dda2a55977c71ef1818df85957a28 Mon Sep 17 00:00:00 2001 From: Adam Frisby Date: Sun, 13 Sep 2026 13:01:02 +0000 Subject: [PATCH] Test selection: enforce coverage-nested subset (Mode=coverage) with ladder Adds Audit:TestSelection:Mode=coverage that executes only the tests whose recorded per-test coverage intersects the changed lines, nested inside the project-graph superset (result is always a subset: coverage shrinks, never grows). Fallback ladder coverage -> project-graph -> all on missing/stale data, selector error, or global-target touch; Mode=all stays the default kill-switch and hot-reloads via IOptionsMonitor. Shadow-before-enforce and structural full-suite-on-main (IRequiredBuildVerifier takes no ITestSelector) are preserved; enforcing modes are opt-in after the soundness gate reports readyForEnforcement for the matching selector. CodeyBox-Prompt-Revision: 1 Co-Authored-By: CodeyBox --- docs/quality/audit.md | 57 +- docs/quality/test-selection.md | 26 +- src/CodeyBox.Api/Program.cs | 25 +- src/CodeyBox.Audit.Shell/DotnetTestAuditor.cs | 66 ++- src/CodeyBox.Core/CoverageTestSelector.cs | 142 +++-- src/CodeyBox.Core/TestSelectionMode.cs | 19 +- src/CodeyBox.Core/TestSelectionShadow.cs | 8 +- src/CodeyBox.Core/TestSelectionTelemetry.cs | 14 +- .../CoverageEnforcementTests.cs | 494 ++++++++++++++++++ .../CoverageTestSelectionTests.cs | 40 +- .../TestSelectionTelemetryTests.cs | 6 +- 11 files changed, 783 insertions(+), 114 deletions(-) create mode 100644 tests/CodeyBox.Tests/CoverageEnforcementTests.cs diff --git a/docs/quality/audit.md b/docs/quality/audit.md index 88740086..0255538e 100644 --- a/docs/quality/audit.md +++ b/docs/quality/audit.md @@ -271,10 +271,12 @@ Capability: `None`. ### Test selection (`Audit:TestSelection`) `csharp:test-pass` can narrow `dotnet test` to the tests a change may affect. -Narrowing ships **advisory/shadow only**: the selector computes the subset it -*WOULD* run, the **full suite still runs**, and per-run telemetry plus a shadow -record capture whether any deselected test failed. No ticket in this sequence -skips a test. +New selectors first ship **advisory/shadow only** (`coverage-shadow`): the +selector computes the subset it *WOULD* run, the **full suite still runs**, +and per-run telemetry plus a shadow record capture whether any deselected +test failed. Real skipping is gated on the soundness gate below reporting +`readyForEnforcement`, after which operators may opt into the enforcing +`project-graph` or `coverage` modes. **Selection scopes (`all` | `project-graph` | `coverage`)** — the three layers, innermost first: @@ -285,22 +287,30 @@ innermost first: - `project-graph` (`ProjectGraphTestSelector`): maps each changed file to its owning MSBuild project and selects the baseline's precomputed affected tests, plus tests defined in the changed files. -- `coverage` (`CoverageTestSelector`): refines the project-graph superset by - per-test coverage intersection — and ALWAYS also selects tests defined in - changed files, tests with NO coverage record (new/uninstrumented), and - everything the project-graph layer picks. Coverage only refines WITHIN that - superset: the result is a union, never less. +- `coverage` (`CoverageTestSelector`): intersects the changed lines with + per-test recorded coverage, NESTED INSIDE the project-graph superset — the + executed set only shrinks that superset, never grows beyond it — and ALWAYS + keeps (within the superset) tests defined in changed files and tests with NO + coverage record (new/uninstrumented). A change no recorded coverage + intersects descends the fallback ladder to the project-graph rung. The live config knob is `Audit:TestSelection:Mode` (`all` | -`coverage-shadow`, case-insensitive, hot-reloaded via `IOptionsMonitor`; an -unrecognised value fails fast at load). `coverage-shadow` runs the -`project-graph` + `coverage` layers advisorially and records the verdict; `all` -runs neither. `csharp:test-pass` consults the configured `ITestSelector` only -in `coverage-shadow` mode, and only on paper: it computes the advisory -selection, still runs the FULL suite (the narrowed `--filter` argv is recorded, -never executed), parses the full run's failed tests, and emits a shadow record -(`safe-for-this-run` / `unsafe-skips-observed` / `full-suite` / -`unverifiable`). +`coverage-shadow` | `project-graph` | `coverage`, case-insensitive, +hot-reloaded via `IOptionsMonitor`; an unrecognised value fails fast at load). +`coverage-shadow` runs the `project-graph` + `coverage` layers advisorially +and records the verdict; `project-graph` enforces the superset rung; +`coverage` enforces the nested coverage subset with fallback down the +coverage → project-graph → all ladder; `all` runs neither. Enable an enforcing +mode only after the soundness gate below reports `readyForEnforcement: true` +for that selector. `csharp:test-pass` consults the configured `ITestSelector` +only in these selection modes, and only for per-item audit runs: the +merge/release verification path (`IRequiredBuildVerifier` / +`process:required-build`) takes no selector dependency and always runs the +full surface. In `coverage-shadow` mode the advisory selection is computed on +paper only: the FULL suite still runs (the narrowed `--filter` argv is +recorded, never executed), the full run's failed tests are parsed, and a +shadow record is emitted (`safe-for-this-run` / `unsafe-skips-observed` / +`full-suite` / `unverifiable`). **Fallback ladder** — running MORE tests is always safe, so ANY uncertainty resolves to the full suite, in this order: unknown/empty changeset → no @@ -309,13 +319,18 @@ than `MaxBaselineAge`) → global-target touch (`Directory.Build.*`, `Directory.Packages.props`, `global.json`, `NuGet.Config`, `CodeyBox.slnx`, `.github/workflows/`) → whole-file change (no line granularity) → changed test file (may define unrecorded tests) → changed file no record references → -project-graph superset already full → selector error. Each rung records its -reason in the telemetry `fallbacks` list and the shadow record detail. +change no recorded coverage intersects (coverage rung only — descends to the +project-graph rung rather than the full suite) → project-graph superset +already full → selector error. In `coverage` mode the ladder is structural: +the coverage rung is attempted first, then the project-graph rung, then the +full suite. Each rung records its reason in the telemetry `fallbacks` list +and the shadow record detail. **Full-suite-on-main soundness invariant** — the merge/release path (`IRequiredBuildVerifier` / `process:required-build`) takes NO dependency on the `ITestSelector` seam and always runs the full build/test surface, -regardless of mode. Selectors are advisory for the audit loop only; the gate +regardless of mode. Selectors narrow per-item audit runs only (advisory in +`coverage-shadow`, enforcing in `project-graph`/`coverage`); the gate that certifies `main` cannot narrow. Enforced structurally in code (see `TestSelectorTests`), not by config. diff --git a/docs/quality/test-selection.md b/docs/quality/test-selection.md index 35de134b..93e1d950 100644 --- a/docs/quality/test-selection.md +++ b/docs/quality/test-selection.md @@ -6,8 +6,9 @@ may affect. New selectors first ship **advisory/shadow only** **full suite still runs**, and a shadow record captures whether any deselected test failed. Real skipping is gated on accumulated shadow data showing zero unsafe skips over the calibration window (the soundness gate, -`readyForEnforcement`). The `project-graph` mode below is the first ENFORCING -mode, enabled by operators only after that gate reports ready. +`readyForEnforcement`). The `project-graph` and `coverage` modes below are the +ENFORCING modes, each enabled by operators only after that gate reports ready +for the matching selector. ## Modes (`Audit:TestSelection:Mode`) @@ -16,6 +17,7 @@ mode, enabled by operators only after that gate reports ready. | `all` (default) | Full suite; the emitted command is byte-identical to the legacy path. Instant kill-switch: hot-reloading back to `all` disables all selection. | | `coverage-shadow` | Coverage selector computes the would-be subset; full suite still runs; one structured `test-selection shadow` log line is emitted per run. | | `project-graph` | ENFORCING: the project-graph selector's subset is executed via `--filter`; any selector error, missing/stale data, global-target touch, or ambiguous result falls back to the full suite (fail-safe). Opt in only after the soundness gate reports zero unsafe skips. | +| `coverage` | ENFORCING: the coverage selector's subset — nested inside the project-graph superset (coverage can only shrink it, never grow beyond it) — is executed via `--filter`. Fallback ladder: coverage rung → project-graph rung (the superset) → full suite, on missing/stale coverage data, selector error, or global-target touch. Opt in only after the soundness gate reports zero unsafe skips for the `coverage` selector. | The value is case-insensitive (`coverage_shadow` also parses) and hot-reloads via `IOptionsMonitor`. An unrecognised value fails fast at load. @@ -28,10 +30,13 @@ via `IOptionsMonitor`. An unrecognised value fails fast at load. precomputed affected tests, plus tests defined in the changed files. A change owned by an ALWAYS-FULL project forces the full suite (see below). - **Coverage** (`CoverageTestSelector`): selects tests whose recorded per-test - coverage intersects the changed lines, and ALWAYS also selects tests defined - in changed files, tests with NO coverage record (new/uninstrumented), and - everything the project-graph selector picks. Coverage only refines WITHIN - that superset — the result is a union, never less. + coverage intersects the changed lines, NESTED INSIDE the project-graph + superset — every candidate (coverage hit, test defined in a changed file, + test with NO coverage record) is kept only when the superset already + contains it. Coverage only shrinks that superset, never grows beyond it + (defense in depth against a poisoned/stale coverage map). A change no + recorded coverage intersects carries no signal, so the ladder descends to + the project-graph rung instead of narrowing. Both fall back to the full suite on ANY uncertainty: no/unknown changeset, no baseline, stale baseline, global targets (`Directory.Build.*`, @@ -90,8 +95,9 @@ suite. Size caps (`MaxBaselineBytes`, `MaxBaselineTests`, - **SHADOW-BEFORE-ENFORCE** — `DotnetTestAuditor` executes `BuildInvocation(TestSelection.All, …)` on every shadow run; the narrowed `--filter` argv is computed for the shadow record only, never executed. - The enforcing `project-graph` mode executes the narrowed argv only after the - soundness gate reported zero unsafe skips over the calibration window. + The enforcing `project-graph` and `coverage` modes execute the narrowed argv + only after the soundness gate reported zero unsafe skips over the calibration + window (for the matching selector). - **FULL-SUITE-ON-MAIN** — the merge/release path (`IRequiredBuildVerifier` / `process:required-build`) takes no `ITestSelector` dependency and always runs everything, enforced in code (see `TestSelectorTests`), not config. @@ -125,13 +131,13 @@ Timeline dashboard pages): | Field | Meaning | |-------|---------| -| `mode` | Live selection mode (`All`, `CoverageShadow`, `ProjectGraph`). | +| `mode` | Live selection mode (`All`, `CoverageShadow`, `ProjectGraph`, `Coverage`). | | `selector` | Selector that decided (`coverage`, `project-graph`, or `none` when neither shadow nor enforcement was active). | | `layers` | Layers consulted, innermost first (`["project-graph","coverage"]` for the coverage selector, which refines the project-graph superset; `["project-graph"]` for enforcing project-graph runs). | | `selectedCount` / `totalCount` | WOULD-BE subset / known universe size. `0/0` means the universe was unknown (no baseline) — the dashboard shows "full suite". For a full-suite fallback with a known universe, both equal the universe size. For enforcing runs, the EXECUTED subset / universe size. | | `estimatedSavedFraction` | Proportional estimate: deselected / total in [0,1]. The dashboard multiplies it by the run's `durationMs` (`est. saved 62.5% (~75s)`). Zero for full-suite runs. For shadow runs this is an estimate, not a measurement — the full suite always ran. | | `assessment` | Shadow verdict (`safe-for-this-run` \| `unsafe-skips-observed` \| `full-suite` \| `unverifiable`), plus `enforced-subset` for enforcing runs that executed a narrowed subset (deselected tests were skipped, so no safe/unsafe claim is made; the soundness gate ignores these runs). | -| `fallbacks` | Which fallback-ladder rungs fired (e.g. `no per-test coverage baseline is available`). Empty when the selector narrowed without falling back. | +| `fallbacks` | Which fallback-ladder rungs fired (e.g. `no per-test coverage baseline is available`, or the coverage → project-graph descent marker `project-graph rung:`). Empty when the selector narrowed without falling back. | | `detail` | Operator-facing selection detail (capped at 4000 chars). | See `tests/CodeyBox.Tests/CoverageTestSelectionTests.cs` and diff --git a/src/CodeyBox.Api/Program.cs b/src/CodeyBox.Api/Program.cs index cd084aa0..3b7c6c39 100644 --- a/src/CodeyBox.Api/Program.cs +++ b/src/CodeyBox.Api/Program.cs @@ -327,7 +327,7 @@ void ConfigureResource(ResourceBuilder r) .Bind(builder.Configuration.GetSection(TestSelectionOptions.SectionName)) .Validate( static opts => TestSelectionModeParser.TryParse(opts.Mode, out _), - $"{TestSelectionOptions.SectionName}:Mode must be one of: all, coverage-shadow, project-graph"); + $"{TestSelectionOptions.SectionName}:Mode must be one of: all, coverage-shadow, project-graph, coverage"); // Coverage-guided selection knobs (Audit:TestSelection:Coverage). Bound through // AddOptions so IOptionsMonitor hot-reloads the // baseline location, age/size caps, and global targets without a restart, with @@ -2617,13 +2617,19 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) // from IOptionsMonitor on every call (hot-reload) and dispatches to the selector // registered for that mode; the default 'all' maps to RunAllTestSelector, whose // TestSelection.All keeps the emitted dotnet-test command byte-identical to the -// legacy path. 'coverage-shadow' maps to CoverageTestSelector, which refines the +// legacy path. 'coverage-shadow' maps to CoverageTestSelector, which narrows the // project-graph superset by per-test coverage — ADVISORY ONLY: the per-item // csharp:test-pass runner computes the decision, still runs the full suite, and // emits a shadow record (SHADOW-BEFORE-ENFORCE). 'project-graph' maps to // ProjectGraphTestSelector and is ENFORCING: the per-item csharp:test-pass runner // executes only the selected subset (fail-safe fallback to the full suite on any -// error or ambiguous result). The merge/release verification +// error or ambiguous result). 'coverage' maps to the same CoverageTestSelector +// instance and is ENFORCING: the runner executes only the coverage-narrowed +// subset nested inside the project-graph superset, descending the +// coverage → project-graph → all fallback ladder on any error or ambiguous +// result. Opt into either enforcing mode only after the soundness gate +// (GET /audit/test-selection/soundness, selector=project-graph or coverage) +// reports readyForEnforcement. The merge/release verification // path (IRequiredBuildVerifier / process:required-build) deliberately takes NO // dependency on this seam: it always verifies the full build/test surface // regardless of Mode. @@ -2632,14 +2638,16 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) var modeMonitor = sp.GetRequiredService>(); var coverageOptionsMonitor = sp.GetRequiredService>(); var projectGraph = new ProjectGraphTestSelector(() => coverageOptionsMonitor.CurrentValue, TimeProvider.System); + var coverage = new CoverageTestSelector( + projectGraph, + () => coverageOptionsMonitor.CurrentValue, + TimeProvider.System); var selectorsByMode = new Dictionary { [TestSelectionMode.All] = new RunAllTestSelector(), - [TestSelectionMode.CoverageShadow] = new CoverageTestSelector( - projectGraph, - () => coverageOptionsMonitor.CurrentValue, - TimeProvider.System), + [TestSelectionMode.CoverageShadow] = coverage, [TestSelectionMode.ProjectGraph] = projectGraph, + [TestSelectionMode.Coverage] = coverage, }; return new ConfiguredTestSelector( () => TestSelectionModeParser.Parse(modeMonitor.CurrentValue.Mode), @@ -2649,7 +2657,8 @@ static Func DotnetTestRunOptionsAccessor(IServiceProvider sp) // threaded into every csharp:test-pass runner the preset catalogs build. // Mode=all (the default) is an instant kill-switch: the runner checks the live // mode on every run and runs the full suite for anything but coverage-shadow -// (advisory shadow) or project-graph (enforcing subset). +// (advisory shadow), project-graph (enforcing subset), or coverage (enforcing +// coverage-nested subset). builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => new TestSelectionShadowConfig { diff --git a/src/CodeyBox.Audit.Shell/DotnetTestAuditor.cs b/src/CodeyBox.Audit.Shell/DotnetTestAuditor.cs index a5b045f2..967e8fb3 100644 --- a/src/CodeyBox.Audit.Shell/DotnetTestAuditor.cs +++ b/src/CodeyBox.Audit.Shell/DotnetTestAuditor.cs @@ -95,7 +95,13 @@ public async Task RunAsync( if (mode == TestSelectionMode.CoverageShadow) return await RunWithShadowAsync(sandbox, workingDirectory, context, shadow, ct).ConfigureAwait(false); if (mode == TestSelectionMode.ProjectGraph) - return await RunWithProjectGraphEnforcementAsync(sandbox, workingDirectory, context, shadow, ct).ConfigureAwait(false); + return await RunWithEnforcementAsync( + sandbox, workingDirectory, context, shadow, + ProjectGraphTestSelector.SelectorName, ct).ConfigureAwait(false); + if (mode == TestSelectionMode.Coverage) + return await RunWithEnforcementAsync( + sandbox, workingDirectory, context, shadow, + CoverageTestSelector.SelectorName, ct).ConfigureAwait(false); } var full = await RunFullAsync(sandbox, workingDirectory, context, ct).ConfigureAwait(false); return full with { TestSelection = TestSelectionTelemetryComputer.FullSuiteWithoutShadow(ResolveModeName(shadow)) }; @@ -249,19 +255,26 @@ private static TestSelectionShadowRecord BuildShadowRecord( } /// - /// ENFORCING project-graph selection (Audit:TestSelection:Mode=project-graph): - /// resolves the affected tests and executes ONLY that subset via - /// --filter. Fail-safe: any selector error, unreadable options, - /// unknown base ref, an ambiguous result (empty/blank filters), a - /// filter-build failure, or a narrowed run that executes zero tests falls - /// back to the full suite. The merge/release path never reaches here — it - /// takes no ITestSelector dependency by construction. + /// ENFORCING selection (Audit:TestSelection:Mode=project-graph or + /// coverage): resolves the affected tests and executes ONLY that + /// subset via --filter. The + /// stamps the telemetry selector (and its layers): the project-graph rung + /// executes the superset, while the coverage rung executes the + /// coverage-narrowed subset nested inside that superset — or, when the + /// coverage rung falls back (), + /// the superset itself, with the fired rung recorded in telemetry + /// fallbacks. Fail-safe: any selector error, unreadable options, unknown + /// base ref, an ambiguous result (empty/blank filters), a filter-build + /// failure, or a narrowed run that executes zero tests falls back to the + /// full suite. The merge/release path never reaches here — it takes no + /// ITestSelector dependency by construction. /// - private async Task RunWithProjectGraphEnforcementAsync( + private async Task RunWithEnforcementAsync( ISandbox sandbox, string workingDirectory, AuditContext context, TestSelectionShadowConfig shadow, + string enforcedSelectorName, CancellationToken ct) { var changedFiles = await TestSelectionShadowIO.GetChangedFilesAsync( @@ -274,7 +287,9 @@ private async Task RunWithProjectGraphEnforcementAsync( var universe = baseline.Baseline is null ? (IReadOnlyList)[] : [.. baseline.Baseline.Tests.Keys]; - var modeName = TestSelectionMode.ProjectGraph.ToString(); + var modeName = enforcedSelectorName == CoverageTestSelector.SelectorName + ? TestSelectionMode.Coverage.ToString() + : TestSelectionMode.ProjectGraph.ToString(); TestSelectionDecision decision; if (string.IsNullOrWhiteSpace(context.BaseBranch)) @@ -307,7 +322,7 @@ private async Task RunWithProjectGraphEnforcementAsync( } var full = await RunFullAsync(sandbox, workingDirectory, context, ct).ConfigureAwait(false); var telemetry = TestSelectionTelemetryComputer.FromEnforcedSelection( - modeName, ProjectGraphTestSelector.SelectorName, decision, universe.Count, detail); + modeName, enforcedSelectorName, decision, universe.Count, detail); return full with { TestSelection = telemetry }; } @@ -321,7 +336,7 @@ private async Task RunWithProjectGraphEnforcementAsync( var buildFailure = detail + $" | filter build failed ({ex.GetType().Name}: {TruncateForDetail(ex.Message)})"; var full = await RunFullAsync(sandbox, workingDirectory, context, ct).ConfigureAwait(false); var fallbackTelemetry = TestSelectionTelemetryComputer.FromEnforcedSelection( - modeName, ProjectGraphTestSelector.SelectorName, + modeName, enforcedSelectorName, new TestSelectionDecision(TestSelection.All, buildFailure), universe.Count, buildFailure); return full with { TestSelection = fallbackTelemetry }; @@ -333,16 +348,29 @@ private async Task RunWithProjectGraphEnforcementAsync( var zeroTests = detail + " | narrowed run executed zero tests; fell back to the full suite"; var full = await RunFullAsync(sandbox, workingDirectory, context, ct).ConfigureAwait(false); var fallbackTelemetry = TestSelectionTelemetryComputer.FromEnforcedSelection( - modeName, ProjectGraphTestSelector.SelectorName, + modeName, enforcedSelectorName, new TestSelectionDecision(TestSelection.All, zeroTests), universe.Count, zeroTests); return full with { TestSelection = fallbackTelemetry }; } var enforcedTelemetry = TestSelectionTelemetryComputer.FromEnforcedSelection( - modeName, ProjectGraphTestSelector.SelectorName, decision, universe.Count, detail); + modeName, enforcedSelectorName, decision, universe.Count, detail, + RungFallbacks(decision)); return narrowed with { TestSelection = enforcedTelemetry }; } + /// + /// Ladder-rung attribution for a narrowed enforcing run: when the coverage + /// selector descended to its project-graph rung, the coverage reason rides + /// in telemetry fallbacks so operators see which rung fired. Any other + /// narrowed run narrowed without falling back. + /// + private static IReadOnlyList? RungFallbacks(TestSelectionDecision decision) + => decision.Justification.Contains( + CoverageTestSelector.ProjectGraphRungMarker, StringComparison.Ordinal) + ? [decision.Justification] + : null; + private static string TruncateForDetail(string message, int maxChars = 200) { if (string.IsNullOrWhiteSpace(message)) @@ -500,10 +528,12 @@ public sealed record DotnetTestAuditorOptions /// Test-selection configuration. When set, the live mode decides the run: /// coverage-shadow computes the selector's advisory decision, still /// executes the FULL suite, and emits a shadow record (SHADOW-BEFORE-ENFORCE); - /// project-graph executes ONLY the selector's subset (enforcing), - /// falling back to the full suite on any error or ambiguous result; - /// all (or unset) runs the full suite. Null (the default) disables - /// selection — byte-identical legacy runs. + /// project-graph executes ONLY the project-graph subset (enforcing), + /// coverage executes ONLY the coverage subset nested inside that + /// superset (enforcing, with fallback down the coverage → project-graph → + /// all ladder), each falling back to the full suite on any error or + /// ambiguous result; all (or unset) runs the full suite. Null (the + /// default) disables selection — byte-identical legacy runs. /// public TestSelectionShadowConfig? Shadow { get; init; } } diff --git a/src/CodeyBox.Core/CoverageTestSelector.cs b/src/CodeyBox.Core/CoverageTestSelector.cs index 447c2372..9d19b472 100644 --- a/src/CodeyBox.Core/CoverageTestSelector.cs +++ b/src/CodeyBox.Core/CoverageTestSelector.cs @@ -13,16 +13,21 @@ public sealed record CoverageSelection( /// /// Pure core of coverage-guided selection. Given the changed lines, selects the -/// tests whose recorded per-test coverage intersects them, and ALWAYS also -/// selects: the tests defined in the changed files, the tests with NO coverage -/// record (new/uninstrumented), and the project-graph superset passed in. -/// Coverage may only refine WITHIN that superset — the result is always a -/// superset of it, never less. +/// tests whose recorded per-test coverage intersects them, NESTED INSIDE the +/// project-graph superset passed in: every candidate (coverage hit, test +/// defined in a changed file, test with NO coverage record) is kept only when +/// it is already a member of that superset. Coverage can only shrink the +/// superset, never grow beyond it — a poisoned or stale coverage map cannot +/// widen the executed set past the project-graph bound (defense in depth). /// /// Fail-safe: a missing or stale baseline, an unknown changeset, a /// global target, a whole-file change, a changed test file (which may define -/// unrecorded tests), or a changed file no coverage record references all -/// resolve to the full suite. +/// unrecorded tests), a changed file no coverage record references, a change +/// no recorded coverage intersects (no signal — the project-graph rung owns +/// it), or zero coverage hits all resolve to the full suite at THIS layer. +/// The then descends the fallback ladder +/// (coverage → project-graph → all) rather than running the full suite +/// directly when the superset still narrows. /// public static class CoverageSelectionCore { @@ -79,14 +84,17 @@ public static CoverageSelection SelectTests( } var changedFilesSet = new HashSet(changedLines.Keys, StringComparer.Ordinal); - var selected = new SortedSet(superset.Tests, StringComparer.Ordinal); + var selected = new SortedSet(StringComparer.Ordinal); var viaCoverage = 0; var viaDefiningFile = 0; var viaNoRecord = 0; foreach (var (name, entry) in baseline.Tests) { - if (selected.Contains(name)) + // Nested-inside-superset: a test outside the project-graph bound is + // never added, however its coverage reads. The superset's soundness + // is the floor; the shadow/soundness gate validates the narrowing. + if (!superset.Tests.Contains(name)) continue; if (entry.Covers.Count == 0) @@ -111,10 +119,19 @@ public static CoverageSelection SelectTests( } } + if (viaCoverage == 0) + { + return Full( + "no recorded per-test coverage intersects the changed lines " + + $"({selected.Count} must-include test(s) inside the superset carry no signal)"); + } + + var deselected = Math.Max(0, superset.Tests.Count - selected.Count); return new CoverageSelection(false, selected, string.Create( CultureInfo.InvariantCulture, $"{selected.Count} test(s) ({viaCoverage} via coverage, " + - $"{viaDefiningFile} defined in changed files, {viaNoRecord} without a coverage record)")); + $"{viaDefiningFile} defined in changed files, {viaNoRecord} without a coverage record; " + + $"{deselected} project-graph test(s) not covering the change deselected)")); } private static CoverageSelection Full(string reason) @@ -153,19 +170,29 @@ private static bool IntersectsChanged( } /// -/// Coverage-guided regression-test selector. Refines the project-graph -/// superset () by coverage intersection -/// while preserving every test the superset picked: the emitted filters are -/// the superset's filters verbatim plus the coverage/must-include test names, -/// so the result is never less than the superset. Falls back to -/// on any uncertainty — see -/// . +/// Coverage-guided regression-test selector. Narrows the project-graph +/// superset () by coverage intersection: +/// the emitted filters are always a subset of that superset — coverage can +/// only shrink it, never grow beyond it. Implements the fallback ladder +/// coverage → project-graph → all: when the coverage rung cannot narrow +/// (missing/stale data, global target, no intersecting coverage, selector +/// error) but the superset still narrows, the superset decision is returned +/// verbatim; only when both rungs fail does the selector fall back to +/// . /// public sealed class CoverageTestSelector : ITestSelector { /// Selector name used in justifications and shadow records. public const string SelectorName = "coverage"; + /// + /// Marker stamped into a project-graph-rung justification: the coverage + /// rung fell back and the superset decision is executed verbatim. The + /// enforcing auditor reads this (exact ordinal match) to attribute the + /// fallback in per-run telemetry. + /// + public const string ProjectGraphRungMarker = "project-graph rung:"; + private readonly ITestSelector _supersetSelector; private readonly Func _optionsProvider; private readonly TimeProvider _clock; @@ -187,7 +214,19 @@ public TestSelectionDecision Select(TestSelectionRequest request) { ArgumentNullException.ThrowIfNull(request); - var supersetDecision = _supersetSelector.Select(request); + TestSelectionDecision supersetDecision; + try + { + supersetDecision = _supersetSelector.Select(request); + } + catch (Exception ex) + { + // Fail-safe: a broken superset selector bottoms the ladder at the + // full run — there is no narrower rung to trust. + return new TestSelectionDecision( + TestSelection.All, + $"{SelectorName}: full suite (superset selector error ({ex.GetType().Name}))"); + } if (supersetDecision.Selection.IsAll) { return new TestSelectionDecision( @@ -195,37 +234,74 @@ public TestSelectionDecision Select(TestSelectionRequest request) $"{SelectorName}: full suite (superset selector chose the full suite: {supersetDecision.Justification})"); } - var options = _optionsProvider(); + CoverageTestSelectionOptions options; + try + { + options = _optionsProvider(); + } + catch (Exception) + { + // The coverage rung cannot read its knobs, but the superset + // decision above already narrowed without them — descend to it. + return ProjectGraphRung(supersetDecision, "selection options unavailable"); + } + + // A superset carrying raw filter expressions (operators the bare-name + // set cannot express) cannot be provably nested inside — execute it + // verbatim rather than risk a false "shrunk" claim. + if (HasRawExpressions(supersetDecision.Selection)) + return ProjectGraphRung(supersetDecision, "superset carries raw filter expressions"); + + var utcNow = _clock.GetUtcNow(); var superset = ProjectGraphSelectorCore.SelectTests( request.ChangedFiles, request.Baseline, options, - _clock.GetUtcNow(), + utcNow, request.CurrentCommit); var resolved = CoverageSelectionCore.SelectTests( request.ChangedFiles, request.Baseline, options, - _clock.GetUtcNow(), + utcNow, superset, request.CurrentCommit); - if (resolved.IsFullSuite) + if (!resolved.IsFullSuite) { + foreach (var test in resolved.Tests) + { + if (!supersetDecision.Selection.Filters.Contains(test)) + { + // Structural defense-in-depth tripwire: the core promises a + // subset of the recomputed superset; the executed superset + // decision must agree. Any drift falls down the ladder. + return ProjectGraphRung(supersetDecision, "coverage result escapes the superset"); + } + } + return new TestSelectionDecision( - TestSelection.All, - $"{SelectorName}: full suite ({resolved.Reason})"); + new TestSelection([.. resolved.Tests]), + $"{SelectorName}: {resolved.Reason}; superset: {supersetDecision.Justification}"); } - // Preserve the superset's filters verbatim (they may carry raw - // expressions a bare-name set cannot express), then add the - // coverage/must-include names. Union — never less than the superset. - var filters = new SortedSet(supersetDecision.Selection.Filters, StringComparer.Ordinal); - foreach (var test in resolved.Tests) - filters.Add(test); + if (superset.IsFullSuite) + { + return new TestSelectionDecision( + TestSelection.All, + $"{SelectorName}: full suite ({resolved.Reason}); superset: {superset.Reason}"); + } - return new TestSelectionDecision( - new TestSelection([.. filters]), - $"{SelectorName}: {resolved.Reason}; superset: {supersetDecision.Justification}"); + return ProjectGraphRung(supersetDecision, resolved.Reason); } + + private static TestSelectionDecision ProjectGraphRung( + TestSelectionDecision supersetDecision, string coverageReason) + => new( + supersetDecision.Selection, + $"{SelectorName}: full suite ({coverageReason}); {ProjectGraphRungMarker} {supersetDecision.Justification}"); + + private static bool HasRawExpressions(TestSelection selection) + => selection.Filters.Any(f => + f.Contains('=', StringComparison.Ordinal) || f.Contains('~', StringComparison.Ordinal)); } diff --git a/src/CodeyBox.Core/TestSelectionMode.cs b/src/CodeyBox.Core/TestSelectionMode.cs index 15960e84..6089f454 100644 --- a/src/CodeyBox.Core/TestSelectionMode.cs +++ b/src/CodeyBox.Core/TestSelectionMode.cs @@ -27,6 +27,20 @@ public enum TestSelectionMode /// ambiguous result falls back to the full suite (fail-safe). /// ProjectGraph, + + /// + /// Enforcing coverage selection: the coverage selector narrows the executed + /// dotnet test run to the tests whose recorded per-test coverage + /// intersects the changed lines, NESTED INSIDE the project-graph superset + /// (the result is always a subset of that superset — coverage can only + /// shrink it, never grow beyond it). Fallback ladder: coverage rung + /// (narrowed) → project-graph rung (the superset) → full suite. + /// Opt-in only after the soundness gate reports zero unsafe skips for the + /// coverage selector over the calibration window. Any selector error, + /// missing/stale data, global-target touch, or ambiguous result falls back + /// down the ladder to the full suite (fail-safe). + /// + Coverage, } /// @@ -85,6 +99,9 @@ public static bool TryParse(string? value, out TestSelectionMode mode) case "projectgraph": mode = TestSelectionMode.ProjectGraph; return true; + case "coverage": + mode = TestSelectionMode.Coverage; + return true; default: return false; } @@ -100,7 +117,7 @@ public static TestSelectionMode Parse(string? value) ? mode : throw new FormatException(string.Create( CultureInfo.InvariantCulture, - $"Unknown {TestSelectionOptions.SectionName}:Mode value '{value}'. Valid modes: all, coverage-shadow, project-graph.")); + $"Unknown {TestSelectionOptions.SectionName}:Mode value '{value}'. Valid modes: all, coverage-shadow, project-graph, coverage.")); } /// diff --git a/src/CodeyBox.Core/TestSelectionShadow.cs b/src/CodeyBox.Core/TestSelectionShadow.cs index 329a4693..93550335 100644 --- a/src/CodeyBox.Core/TestSelectionShadow.cs +++ b/src/CodeyBox.Core/TestSelectionShadow.cs @@ -231,9 +231,11 @@ public sealed record TestSelectionShadowConfig /// /// Live mode reader (backed by IOptionsMonitor). The shadow runs - /// ONLY for ; the enforcing - /// project-graph run applies ONLY for - /// ; every other + /// ONLY for ; an enforcing + /// subset run applies ONLY for + /// (project-graph subset) or + /// (coverage subset nested inside the project-graph superset, with + /// fallback down the coverage → project-graph → all ladder); every other /// mode — including the all kill-switch — runs the full suite with /// no selection. /// diff --git a/src/CodeyBox.Core/TestSelectionTelemetry.cs b/src/CodeyBox.Core/TestSelectionTelemetry.cs index 2ce75e82..2711995e 100644 --- a/src/CodeyBox.Core/TestSelectionTelemetry.cs +++ b/src/CodeyBox.Core/TestSelectionTelemetry.cs @@ -187,13 +187,18 @@ public static TestSelectionTelemetry FullSuiteWithoutShadow(string mode) /// tests were not executed, so no safe/unsafe verdict can be claimed and /// the soundness gate (which only counts safe/unsafe) ignores these runs. /// Full-suite fallbacks report full-suite with the fallback reason. + /// An optional list records intermediate + /// ladder rungs that fired on the way to a narrowed run (e.g. the coverage + /// rung falling back to the executed project-graph superset); entries are + /// truncated and capped like any fallback. /// public static TestSelectionTelemetry FromEnforcedSelection( string mode, string selectorName, TestSelectionDecision decision, int universeCount, - string detail) + string detail, + IReadOnlyList? fallbacks = null) { ArgumentNullException.ThrowIfNull(decision); ArgumentNullException.ThrowIfNull(detail); @@ -230,7 +235,12 @@ public static TestSelectionTelemetry FromEnforcedSelection( TotalCount = universeCount, EstimatedSavedFraction = fraction, Assessment = AssessmentEnforced, - Fallbacks = [], + Fallbacks = fallbacks is null + ? [] + : [.. fallbacks + .Where(f => !string.IsNullOrWhiteSpace(f)) + .Select(f => Truncate(f, MaxFallbackChars)) + .Take(MaxFallbacks)], Detail = Truncate(detail, MaxDetailChars), }; } diff --git a/tests/CodeyBox.Tests/CoverageEnforcementTests.cs b/tests/CodeyBox.Tests/CoverageEnforcementTests.cs new file mode 100644 index 00000000..b6e3ca74 --- /dev/null +++ b/tests/CodeyBox.Tests/CoverageEnforcementTests.cs @@ -0,0 +1,494 @@ +using CodeyBox.Api; +using CodeyBox.Audit.Shell; +using CodeyBox.Core; +using CodeyBox.Orchestrator; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; + +namespace CodeyBox.Tests; + +/// +/// Enforcing coverage test selection: with +/// Audit:TestSelection:Mode=coverage the per-item +/// csharp:test-pass run executes only the tests whose recorded +/// per-test coverage intersects the changed lines, NESTED INSIDE the +/// project-graph superset (coverage can only shrink it, never grow beyond +/// it). The fallback ladder runs coverage → project-graph → all: a change +/// with no intersecting coverage but a narrowing superset executes the +/// superset, while a shared-root change, stale baseline, selector error, or +/// the merge/release path runs the full suite. +/// +public sealed class CoverageEnforcementTests +{ + private static readonly string[] BaseDotnetTest = ["dotnet", "test", "--no-build"]; + + [Theory] + [InlineData("coverage", TestSelectionMode.Coverage)] + [InlineData("Coverage", TestSelectionMode.Coverage)] + [InlineData(" coverage ", TestSelectionMode.Coverage)] + [InlineData("COVERAGE", TestSelectionMode.Coverage)] + public void ModeParser_ParsesCoverage(string value, TestSelectionMode expected) + { + Assert.True(TestSelectionModeParser.TryParse(value, out var mode)); + Assert.Equal(expected, mode); + Assert.Equal(expected, TestSelectionModeParser.Parse(value)); + } + + [Fact] + public async Task EnforcingRun_PrivateMethodChange_RunsOnlyCoveringTests() + { + // End-to-end acceptance: a change to one executable line runs only the + // tests that cover it — never the superset remainder, never tests + // outside the superset. + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 10), + CoverageBaseline(), + "Passed! - Failed: 0, Passed: 1"); + var auditor = EnforcingRunner(sink); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.Contains("--filter", testArgv); + var filter = testArgv[((List)[.. testArgv]).IndexOf("--filter") + 1]; + Assert.Contains("Ns.Leaf.LeafTests", filter); + Assert.DoesNotContain("Ns.Leaf.OtherTests", filter); + Assert.DoesNotContain("Ns.Leaf.NewTests", filter); + Assert.DoesNotContain("Ns.Other.UnrelatedTests", filter); + + Assert.NotNull(result.TestSelection); + Assert.Equal(TestSelectionMode.Coverage.ToString(), result.TestSelection.Mode); + Assert.Equal(CoverageTestSelector.SelectorName, result.TestSelection.Selector); + Assert.Equal( + [ProjectGraphTestSelector.SelectorName, CoverageTestSelector.SelectorName], + result.TestSelection.Layers); + Assert.Equal(TestSelectionTelemetryComputer.AssessmentEnforced, result.TestSelection.Assessment); + Assert.Equal(1, result.TestSelection.SelectedCount); + Assert.Equal(4, result.TestSelection.TotalCount); + Assert.Empty(result.TestSelection.Fallbacks); + } + + [Fact] + public async Task EnforcingRun_NoIntersectingCoverage_FallsBackToProjectGraph() + { + // The changed line is referenced but covered by nothing: the coverage + // rung carries no signal, so the ladder descends to the superset. + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 99), + CoverageBaseline(), + "Passed! - Failed: 0, Passed: 2"); + var auditor = EnforcingRunner(sink); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.Contains("--filter", testArgv); + var filter = testArgv[((List)[.. testArgv]).IndexOf("--filter") + 1]; + Assert.Contains("Ns.Leaf.LeafTests", filter); + Assert.Contains("Ns.Leaf.OtherTests", filter); + Assert.DoesNotContain("Ns.Other.UnrelatedTests", filter); + + Assert.NotNull(result.TestSelection); + Assert.Equal(TestSelectionTelemetryComputer.AssessmentEnforced, result.TestSelection.Assessment); + Assert.Equal(2, result.TestSelection.SelectedCount); + Assert.Equal(4, result.TestSelection.TotalCount); + var fallback = Assert.Single(result.TestSelection.Fallbacks); + Assert.Contains(CoverageTestSelector.ProjectGraphRungMarker, fallback); + } + + [Fact] + public async Task EnforcingRun_FileUnknownToCoverageButKnownToGraph_FallsBackToProjectGraph() + { + // Absent per-file coverage (no record references the file) with a + // narrowing project-graph superset executes the superset, not the + // full suite. + var sink = new InMemoryTestSelectionShadowSink(); + var baseline = $$""" + { + "format": "codeybox-test-selection-baseline/1", + "commit": "abc123", + "producedAtUtc": "{{DateTimeOffset.UtcNow.AddHours(-1):O}}", + "fileProject": { "src/Brand/New.cs": "src/Brand/Brand.csproj" }, + "projects": { "src/Brand/Brand.csproj": ["Ns.Brand.BrandTests"] }, + "tests": { + "Ns.Brand.BrandTests": { + "file": "tests/Brand.Tests/BrandTests.cs", + "covers": { "src/Brand/Other.cs": [7] } + } + } + } + """; + var sandbox = SandboxFor( + DiffFor("src/Brand/New.cs", startLine: 3), + baseline, + "Passed! - Failed: 0, Passed: 1"); + var auditor = EnforcingRunner(sink); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.Contains("--filter", testArgv); + var filter = testArgv[((List)[.. testArgv]).IndexOf("--filter") + 1]; + Assert.Contains("Ns.Brand.BrandTests", filter); + Assert.NotNull(result.TestSelection); + Assert.Equal(TestSelectionTelemetryComputer.AssessmentEnforced, result.TestSelection.Assessment); + } + + [Fact] + public async Task EnforcingRun_StaleBaseline_FallsBackToFullSuite() + { + // A baseline older than MaxBaselineAge fails BOTH rungs (the ladder + // descends through the project-graph rung, which is stale too). + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 10), + CoverageBaseline(DateTimeOffset.UtcNow.AddDays(-8)), + "Passed!"); + var auditor = EnforcingRunner(sink); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.DoesNotContain("--filter", testArgv); + Assert.Equal(TestSelectionShadowRecord.AssessmentFullSuite, result.TestSelection!.Assessment); + Assert.Contains("stale", result.TestSelection.Detail); + } + + [Fact] + public async Task EnforcingRun_CoreChange_FallsBackToFullSuite() + { + // A change owned by an ALWAYS-FULL project (the shared core contract + // assembly) runs the full suite even with fresh coverage data. + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/CodeyBox.Core/Foo.cs", startLine: 10), + CoverageBaseline(), + "Passed!"); + var auditor = EnforcingRunner(sink); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.DoesNotContain("--filter", testArgv); + Assert.Equal(TestSelectionShadowRecord.AssessmentFullSuite, result.TestSelection!.Assessment); + Assert.Contains("always-full", result.TestSelection.Detail); + } + + [Fact] + public async Task EnforcingRun_SupersetSelectorError_FallsBackToFullSuite() + { + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 10), + CoverageBaseline(), + "Passed!"); + var auditor = new DotnetTestAuditor(new DotnetTestAuditorOptions + { + Name = "csharp:test-pass", + BaseArgv = BaseDotnetTest, + Shadow = new TestSelectionShadowConfig + { + Selector = new CoverageTestSelector( + new ThrowingSelector(), + () => new CoverageTestSelectionOptions(), + TimeProvider.System), + Sink = new InMemoryTestSelectionShadowSink(), + ModeAccessor = () => TestSelectionMode.Coverage, + OptionsAccessor = () => new CoverageTestSelectionOptions(), + }, + }); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.DoesNotContain("--filter", testArgv); + Assert.Equal(TestSelectionShadowRecord.AssessmentFullSuite, result.TestSelection!.Assessment); + } + + [Fact] + public async Task EnforcingRun_CoverageOptionsError_FallsBackToProjectGraph() + { + // The coverage rung cannot read its knobs, but the already-computed + // superset decision narrowed without them — the ladder descends to it. + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 10), + CoverageBaseline(), + "Passed! - Failed: 0, Passed: 2"); + var auditor = new DotnetTestAuditor(new DotnetTestAuditorOptions + { + Name = "csharp:test-pass", + BaseArgv = BaseDotnetTest, + Shadow = new TestSelectionShadowConfig + { + Selector = new CoverageTestSelector( + new ProjectGraphTestSelector(() => new CoverageTestSelectionOptions()), + ThrowingOptions, + TimeProvider.System), + Sink = sink, + ModeAccessor = () => TestSelectionMode.Coverage, + OptionsAccessor = () => new CoverageTestSelectionOptions(), + }, + }); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.Contains("--filter", testArgv); + Assert.NotNull(result.TestSelection); + Assert.Equal(TestSelectionTelemetryComputer.AssessmentEnforced, result.TestSelection.Assessment); + } + + [Fact] + public async Task EnforcingRun_AllMode_IsKillSwitch() + { + var sink = new InMemoryTestSelectionShadowSink(); + var sandbox = SandboxFor( + DiffFor("src/Leaf/A.cs", startLine: 10), + CoverageBaseline(), + "Passed!"); + var auditor = new DotnetTestAuditor(new DotnetTestAuditorOptions + { + Name = "csharp:test-pass", + BaseArgv = BaseDotnetTest, + Shadow = new TestSelectionShadowConfig + { + Selector = new CoverageTestSelector( + new ProjectGraphTestSelector(() => new CoverageTestSelectionOptions()), + () => new CoverageTestSelectionOptions(), + TimeProvider.System), + Sink = sink, + ModeAccessor = () => TestSelectionMode.All, + OptionsAccessor = () => new CoverageTestSelectionOptions(), + }, + }); + + var result = await auditor.RunAsync(sandbox, "/work", ContextFor()); + + Assert.True(result.Passed); + var testArgv = Assert.Single(sandbox.ExecutedArgv, a => a.Count > 1 && a[1] == "test"); + Assert.DoesNotContain("--filter", testArgv); + Assert.Empty(sink.Records); + } + + [Fact] + public void Program_ResolvesCoverageSelector_ForEnforcingMode() + { + using var factory = new EnforcingWiringFactory(new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [$"{TestSelectionOptions.SectionName}:Mode"] = "coverage", + }); + + var selector = factory.Services.GetRequiredService(); + var runner = factory.Services.GetRequiredService(); + var decision = selector.Select(new TestSelectionRequest(runner, "main", [], baseline: null)); + + Assert.True(decision.Selection.IsAll); + Assert.Contains(CoverageTestSelector.SelectorName, decision.Justification); + } + + [Fact] + public void Program_DefaultModeStaysAll() + { + using var factory = new EnforcingWiringFactory(); + var monitor = factory.Services.GetRequiredService>(); + Assert.Equal(TestSelectionModeParser.DefaultModeName, monitor.CurrentValue.Mode); + } + + [Fact] + public async Task RequiredBuildVerification_NeverInvokesSelector_UnderCoverageMode() + { + // FULL-SUITE-ON-MAIN IS STRUCTURAL: even with Mode=coverage the + // merge/release verifier takes no ITestSelector dependency and never + // consults the seam. + var recording = new CountingSelector(); + using var factory = new EnforcingWiringFactory( + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [$"{TestSelectionOptions.SectionName}:Mode"] = "coverage", + }, + services => + { + services.RemoveAll(); + services.AddSingleton(recording); + }); + + var verifier = factory.Services.GetRequiredService(); + var result = await verifier.VerifyAsync(new RequiredBuildVerificationRequest + { + WorkItemId = new WorkItemId(Guid.NewGuid()), + ProjectId = new ProjectId("test-project"), + RepositoryId = "does-not-exist-" + Guid.NewGuid().ToString("N"), + BaseBranch = "main", + WorkBranch = "main", + Phase = "audit", + SandboxPolicy = new RequiredBuildSandboxPolicy(), + }, CancellationToken.None); + + Assert.NotNull(result); + Assert.NotEqual(RequiredBuildVerificationStatus.Passed, result.Status); + Assert.NotEqual(RequiredBuildVerificationStatus.Failed, result.Status); + Assert.Equal(0, recording.Calls); + } + + private static CoverageTestSelectionOptions ThrowingOptions() + => throw new InvalidOperationException("options down"); + + private static DotnetTestAuditor EnforcingRunner(InMemoryTestSelectionShadowSink sink) + => new(new DotnetTestAuditorOptions + { + Name = "csharp:test-pass", + BaseArgv = BaseDotnetTest, + Shadow = new TestSelectionShadowConfig + { + Selector = new CoverageTestSelector( + new ProjectGraphTestSelector(() => new CoverageTestSelectionOptions()), + () => new CoverageTestSelectionOptions(), + TimeProvider.System), + Sink = sink, + ModeAccessor = () => TestSelectionMode.Coverage, + OptionsAccessor = () => new CoverageTestSelectionOptions(), + }, + }); + + private static string DiffFor(string path, int startLine) + => $"diff --git a/{path} b/{path}\n+++ b/{path}\n@@ -0,0 +{startLine},1 @@\n+var x = 1;\n"; + + private static string CoverageBaseline(DateTimeOffset? producedAt = null) + => $$""" + { + "format": "codeybox-test-selection-baseline/1", + "commit": "abc123", + "producedAtUtc": "{{(producedAt ?? DateTimeOffset.UtcNow.AddHours(-1)):O}}", + "fileProject": { + "src/Leaf/A.cs": "src/Leaf/Leaf.csproj", + "src/CodeyBox.Core/Foo.cs": "src/CodeyBox.Core/CodeyBox.Core.csproj" + }, + "projects": { + "src/Leaf/Leaf.csproj": ["Ns.Leaf.LeafTests", "Ns.Leaf.OtherTests"], + "src/CodeyBox.Core/CodeyBox.Core.csproj": ["Ns.Leaf.LeafTests", "Ns.Leaf.OtherTests", "Ns.Other.UnrelatedTests", "Ns.Leaf.NewTests"] + }, + "tests": { + "Ns.Leaf.LeafTests": { + "file": "tests/Leaf.Tests/LeafTests.cs", + "covers": { "src/Leaf/A.cs": [10, 11, 12] } + }, + "Ns.Leaf.OtherTests": { + "file": "tests/Leaf.Tests/OtherTests.cs", + "covers": { "src/Leaf/A.cs": [50] } + }, + "Ns.Leaf.NewTests": { "file": "tests/Leaf.Tests/NewTests.cs", "covers": {} }, + "Ns.Other.UnrelatedTests": { + "file": "tests/Other.Tests/UnrelatedTests.cs", + "covers": { "src/Other.cs": [5] } + } + } + } + """; + + private static FakeSandbox SandboxFor(string diff, string baseline, string testOutput) + => new(exec => + { + if (exec.Argv.Count > 0 && exec.Argv[0] == "git") + return new SandboxExecResult(0, diff, ""); + if (exec.Argv.Count > 0 && exec.Argv[0] == "cat") + return new SandboxExecResult(0, baseline, ""); + return new SandboxExecResult(0, testOutput, ""); + }); + + private static AuditContext ContextFor() + => new(WorkItemId.New(), "work", "main", 1, "prompt"); + + private sealed class ThrowingSelector : ITestSelector + { + public TestSelectionDecision Select(TestSelectionRequest request) + => throw new InvalidOperationException("selector down"); + } + + private sealed class CountingSelector : ITestSelector + { + public int Calls { get; private set; } + + public TestSelectionDecision Select(TestSelectionRequest request) + { + Calls++; + return new TestSelectionDecision(TestSelection.All, "counting selector"); + } + } + + private sealed class FakeSandbox(Func onExec) : ISandbox + { + private readonly Func _onExec = onExec; + public List> ExecutedArgv { get; } = new(); + public string Id => "fake"; + public Task ExecAsync(SandboxExec exec, CancellationToken ct = default) + { + ExecutedArgv.Add(exec.Argv); + return Task.FromResult(_onExec(exec)); + } + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class EnforcingWiringFactory : WebApplicationFactory + { + private readonly string _dbPath = Path.Combine( + Path.GetTempPath(), $"codeybox-coverage-enforcing-wiring-{Guid.NewGuid():N}.db"); + private readonly IReadOnlyDictionary _extra; + private readonly Action? _configureServices; + + public EnforcingWiringFactory( + IReadOnlyDictionary? extra = null, + Action? configureServices = null) + { + _extra = extra ?? new Dictionary(); + _configureServices = configureServices; + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Development"); + builder.ConfigureAppConfiguration((_, cfg) => + { + cfg.Sources.Clear(); + var tmp = Path.GetTempPath(); + var settings = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["CodeyBox:DangerouslyDisableAuth"] = "true", + ["CodeyBox:StateDatabasePath"] = _dbPath, + ["CodeyBox:GitRootDirectory"] = Path.Combine(tmp, $"test-git-{Guid.NewGuid():N}"), + ["CodeyBox:AuditLog:Path"] = Path.Combine(tmp, $"test-log-{Guid.NewGuid():N}-.json"), + ["CodeyBox:AuditLog:AuditPath"] = Path.Combine(tmp, $"test-audit-{Guid.NewGuid():N}-.json"), + ["CodeyBox:AgentStreams:Path"] = Path.Combine(tmp, $"test-agent-streams-{Guid.NewGuid():N}"), + }; + foreach (var kv in _extra) + settings[kv.Key] = kv.Value; + cfg.AddInMemoryCollection(settings); + }); + builder.ConfigureTestServices(services => + { + services.RemoveAll(); + _configureServices?.Invoke(services); + }); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + try { File.Delete(_dbPath); } catch { } + base.Dispose(disposing); + } + } +} diff --git a/tests/CodeyBox.Tests/CoverageTestSelectionTests.cs b/tests/CodeyBox.Tests/CoverageTestSelectionTests.cs index 2535aa7b..a2f0c04a 100644 --- a/tests/CodeyBox.Tests/CoverageTestSelectionTests.cs +++ b/tests/CodeyBox.Tests/CoverageTestSelectionTests.cs @@ -474,7 +474,7 @@ private static CoverageTestSelector NewCoverageSelector(TimeProvider? clock = nu } [Fact] - public void Coverage_SelectsIntersectingPlusMustInclude_WithinSuperset() + public void Coverage_NarrowsSupersetToCoveringTests_NeverGrowsBeyondIt() { var runner = NewRunner(); var decision = NewCoverageSelector().Select(RequestFor( @@ -483,42 +483,47 @@ public void Coverage_SelectsIntersectingPlusMustInclude_WithinSuperset() Assert.False(decision.Selection.IsAll); var selected = decision.Selection.Filters.OrderBy(f => f, StringComparer.Ordinal).ToList(); - // Line 10-11 intersect BarTests; OtherTests rides the project-graph - // superset; NewTests has no coverage record and is always included. - Assert.Equal( - ["Ns.Foo.BarTests", "Ns.Foo.NewTests", "Ns.Foo.OtherTests"], - selected); + // Line 10-11 intersect BarTests only: OtherTests (covers line 50) is + // deselected even though the project-graph superset contains it, and + // NewTests (no coverage record) stays out because it is outside the + // superset — coverage shrinks the superset, never grows beyond it. + Assert.Equal(["Ns.Foo.BarTests"], selected); + Assert.Contains("deselected", decision.Justification); } [Fact] - public void Coverage_NonExecutableChange_KeepsSupersetOnly() + public void Coverage_NoIntersectingCoverage_FallsBackToProjectGraphRung() { // Line 99 is executable nowhere, but the file is referenced — the - // change contributes no coverage hits, so the superset floor stands. + // coverage rung carries no signal, so the ladder descends to the + // project-graph superset instead of the full suite. var decision = NewCoverageSelector().Select(RequestFor( NewRunner(), StandardBaseline(), new TestSelectionChangedFile("src/Foo/Bar.cs", [new ChangedLineRange(99, 1)]))); Assert.False(decision.Selection.IsAll); Assert.Equal( - ["Ns.Foo.BarTests", "Ns.Foo.NewTests", "Ns.Foo.OtherTests"], + ["Ns.Foo.BarTests", "Ns.Foo.OtherTests"], [.. decision.Selection.Filters.OrderBy(f => f, StringComparer.Ordinal)]); + Assert.Contains(CoverageTestSelector.ProjectGraphRungMarker, decision.Justification); } [Fact] - public void Coverage_PreservesSupersetFiltersVerbatim_NeverSelectsLess() + public void Coverage_RawSupersetExpression_ExecutedVerbatim_NeverGrown() { - // A superset emitting a raw expression: the coverage selector must - // preserve it verbatim (never less than the superset). + // A superset emitting a raw expression cannot be provably nested + // inside — the ladder descends to it verbatim and coverage adds + // nothing (never grows beyond the superset). var superset = new RawExpressionSelector("FullyQualifiedName~Flaky"); var selector = new CoverageTestSelector(superset, FreshOptions, new FixedClock(Now)); var decision = selector.Select(RequestFor( NewRunner(), StandardBaseline(), new TestSelectionChangedFile("src/Foo/Bar.cs", [new ChangedLineRange(10, 1)]))); + Assert.False(decision.Selection.IsAll); Assert.Contains("FullyQualifiedName~Flaky", decision.Selection.Filters); - Assert.Contains("Ns.Foo.BarTests", decision.Selection.Filters); - Assert.Contains("Ns.Foo.NewTests", decision.Selection.Filters); + Assert.DoesNotContain("Ns.Foo.BarTests", decision.Selection.Filters); + Assert.Contains(CoverageTestSelector.ProjectGraphRungMarker, decision.Justification); } [Fact] @@ -856,7 +861,12 @@ public async Task ShadowRun_ExecutesFullSuite_EmitsSafeRecord() Assert.Equal(TestSelectionShadowRecord.AssessmentSafe, record.Assessment); Assert.Empty(record.UnsafeSkips); Assert.Contains("--filter", record.WouldBeArgv); - Assert.Equal(["Ns.Foo.UnrelatedTests"], record.DeselectedTests); + // Nested inside the project-graph superset {BarTests, OtherTests}: + // only BarTests covers the changed line, so every other recorded test + // would have been deselected. + Assert.Equal( + ["Ns.Foo.NewTests", "Ns.Foo.OtherTests", "Ns.Foo.UnrelatedTests"], + record.DeselectedTests); } [Fact] diff --git a/tests/CodeyBox.Tests/TestSelectionTelemetryTests.cs b/tests/CodeyBox.Tests/TestSelectionTelemetryTests.cs index 43f27342..0836a600 100644 --- a/tests/CodeyBox.Tests/TestSelectionTelemetryTests.cs +++ b/tests/CodeyBox.Tests/TestSelectionTelemetryTests.cs @@ -167,8 +167,8 @@ public async Task ShadowRunner_AttachesNarrowedTelemetry() Assert.Equal(TestSelectionMode.CoverageShadow.ToString(), telemetry.Mode); Assert.Equal(CoverageTestSelector.SelectorName, telemetry.Selector); Assert.Equal(4, telemetry.TotalCount); - Assert.Equal(3, telemetry.SelectedCount); - Assert.Equal(0.25, telemetry.EstimatedSavedFraction, precision: 9); + Assert.Equal(1, telemetry.SelectedCount); + Assert.Equal(0.75, telemetry.EstimatedSavedFraction, precision: 9); Assert.Equal(TestSelectionShadowRecord.AssessmentSafe, telemetry.Assessment); Assert.Empty(telemetry.Fallbacks); } @@ -185,7 +185,7 @@ public async Task ShadowRunner_UnsafeSkip_SurfacesInTelemetryAssessment() Assert.NotNull(result.TestSelection); var telemetry = result.TestSelection; Assert.Equal(TestSelectionShadowRecord.AssessmentUnsafe, telemetry.Assessment); - Assert.Equal(3, telemetry.SelectedCount); + Assert.Equal(1, telemetry.SelectedCount); Assert.Equal(4, telemetry.TotalCount); }