diff --git a/.claude/tools/append_only_gate.py b/.claude/tools/append_only_gate.py index 654b83d28..384901fc5 100755 --- a/.claude/tools/append_only_gate.py +++ b/.claude/tools/append_only_gate.py @@ -51,12 +51,21 @@ # scratch/roadmap files), and protecting them would make the gate fire on # correct work -- a gate that objects to everything carries exactly as much # information as one that never fires. +# The canonical eight are the ones `.claude/BOOT.md`'s immutability table names +# and `.claude/settings.json` denies `Edit`/`Write`/`MultiEdit` on. The first +# version of this tuple SUBSTITUTED `AGENT_LOG.md` for `IDEAS.md` and kept the +# count at eight, which is exactly why the substitution looked right -- leaving +# a real 1204-line append-only ledger unguarded while `.claude/board/**` +# happily started this workflow on every PR that truncated it. Found by review, +# not by the gate. `AGENT_LOG.md` stays: CLAUDE.md's one-writer rule calls it +# append-only too, so the set is the canonical eight PLUS that one -- nine. PROTECTED = ( ".claude/board/LATEST_STATE.md", ".claude/board/EPIPHANIES.md", ".claude/board/PR_ARC_INVENTORY.md", ".claude/board/STATUS_BOARD.md", ".claude/board/ISSUES.md", + ".claude/board/IDEAS.md", ".claude/board/TECH_DEBT.md", ".claude/board/AGENT_LOG.md", ".claude/board/INTEGRATION_PLANS.md", diff --git a/.claude/tools/citation_decay.py b/.claude/tools/citation_decay.py index d795f48ee..f1f391433 100644 --- a/.claude/tools/citation_decay.py +++ b/.claude/tools/citation_decay.py @@ -57,7 +57,17 @@ Pure stdlib. Usage: python3 .claude/tools/citation_decay.py [paths-or-globs ...] + python3 .claude/tools/citation_decay.py --since BASE python3 .claude/tools/citation_decay.py --self-test + +`--since BASE` is the CI-gating mode: it compares citation verdicts at +`merge-base(BASE, HEAD)` against verdicts at HEAD (the corpus is discovered +by globbing DEFAULT_GLOBS at each revision -- no file list is taken), and +fails ONLY on a citation that is DECAYED at HEAD and was not DECAYED at base. +This replaces an earlier `--added-lines-only BASE ` mode that +filtered by which LINES a PR added -- wrong for this gate's own motivating +case, `EPIPHANIES.md`: a prepend decays citations sitting on UNCHANGED lines +in UNCHANGED files, which an added-lines filter cannot see by construction. """ from __future__ import annotations @@ -239,53 +249,158 @@ def line_of(off: int) -> int: """ -def added_lines(base: str, root: str) -> dict[str, set[int]] | None: - """Line numbers ADDED or MODIFIED by `base...HEAD`, per file. - - Why this exists, and why the gate is useless without it: `EPIPHANIES.md` - alone carries 10 pre-existing decayed citations, and this workspace's - board-hygiene rule means nearly EVERY pull request touches it. A gate - scoped to changed FILES would therefore fail almost every PR for decay it - did not introduce -- and a guard that fires on everything carries exactly - as much information as one that never fires (the `closed_class_guess` - 150/150 defect, root CLAUDE.md). Scoping to changed LINES makes it a - no-new-decay gate: the 124-item backlog on main is a separate deliberate - pass, not every contributor's problem. - - Returns None when the diff cannot be computed, and the caller then FAILS - CLOSED rather than silently checking everything. - """ +def git_merge_base(base: str, root: str) -> str | None: try: out = subprocess.run( - ["git", "diff", "--unified=0", "--diff-filter=d", f"{base}...HEAD"], + ["git", "merge-base", base, "HEAD"], cwd=root, capture_output=True, text=True, check=True, - ).stdout + ).stdout.strip() + return out or None except Exception: return None - per: dict[str, set[int]] = {} - cur: str | None = None - for ln in out.splitlines(): - if ln.startswith("+++ b/"): - cur = ln[6:] - per.setdefault(cur, set()) - elif ln.startswith("@@") and cur is not None: - m = re.search(r"\+(\d+)(?:,(\d+))?", ln) - if m: - start = int(m.group(1)) - count = int(m.group(2) or 1) - per[cur].update(range(start, start + count)) - return per - - -def run(paths: list[str], root: str, only: dict[str, set[int]] | None = None) -> int: + + +class _MergeBaseWorktree: + """A detached worktree at `merge_base_sha`, cleaned up on exit. + + Deliberately the ONLY git command this module runs beyond read-only + `merge-base`/`diff` -- `git worktree add --detach ` never + touches the caller's checked-out tree, unlike `checkout`/`reset`/`clean`. + """ + + def __init__(self, sha: str, root: str): + self.sha = sha + self.root = root + self.dir: str | None = None + + def __enter__(self) -> str | None: + self.dir = tempfile.mkdtemp(prefix="citation-decay-base-") + try: + subprocess.run( + ["git", "worktree", "add", "--detach", self.dir, self.sha], + cwd=self.root, capture_output=True, text=True, check=True, + ) + except Exception: + return None + return self.dir + + def __exit__(self, *exc): + if self.dir is not None: + subprocess.run( + ["git", "worktree", "remove", "--force", self.dir], + cwd=self.root, capture_output=True, text=True, check=False, + ) + return False + + +def collect_citations(root: str) -> dict[tuple, Finding]: + """Every citation under DEFAULT_GLOBS at `root`, keyed by CONTENT. + + Keyed by `(citing relpath, cited path as written, cited start, cited end, + occurrence index)` -- never by line number, since the citing line shifts + on a prepend exactly like the cited one does. The occurrence index + disambiguates two identical citations in one file (same cited span, + same order of appearance), so repeated citations still pair 1:1 between + two revisions instead of colliding into one key. + """ + paths: list[str] = [] + for pat in DEFAULT_GLOBS: + paths.extend(glob.glob(os.path.join(root, pat), recursive=True)) + out: dict[tuple, Finding] = {} + for path in sorted(set(paths)): + if not os.path.isfile(path): + continue + rel = os.path.relpath(path, root).replace(os.sep, "/") + occ: dict[tuple, int] = {} + for f in scan_file(path, root): + base_key = (rel, f.cited, f.start, f.end) + occ[base_key] = occ.get(base_key, -1) + 1 + out[base_key + (occ[base_key],)] = f + return out + + +def since_regression(base: str, root: str): + """Compare citation verdicts at `merge-base(base, HEAD)` vs HEAD. + + Returns (new_decays, preexisting_decays, fixed, None) on success, or + (None, None, None, error_message) when the comparison cannot be made -- + the caller then FAILS CLOSED rather than silently checking everything or + silently passing. + + Why a regression comparison instead of a changed-lines filter: the + citations that decay from an EPIPHANIES.md prepend sit on UNCHANGED lines + in UNCHANGED files (only the file's LATER content moved under them) -- + an --added-lines-only filter structurally cannot see them, which is + exactly the gap this replaces. Scoping to changed FILES instead is + equally wrong the other way: EPIPHANIES.md alone carries pre-existing + decays and nearly every PR touches it, so that would fail almost every + PR for backlog it did not introduce (the `closed_class_guess` 150/150 + defect, root CLAUDE.md: a guard that fires on everything carries exactly + as much information as one that never fires). + """ + merge_base = git_merge_base(base, root) + if merge_base is None: + return None, None, None, ( + f"cannot resolve merge-base('{base}', HEAD) -- refusing to check " + "every citation instead (in CI this usually means a shallow " + "checkout; the workflow needs `fetch-depth: 0`)." + ) + with _MergeBaseWorktree(merge_base, root) as base_dir: + if base_dir is None: + return None, None, None, ( + f"cannot create a worktree at merge-base {merge_base} -- " + "refusing to check every citation instead." + ) + base_map = collect_citations(base_dir) + head_map = collect_citations(root) + + new_decays, preexisting, fixed = [], [], [] + for key, f in head_map.items(): + base_f = base_map.get(key) + base_verdict = base_f.verdict if base_f is not None else None + if f.verdict == DECAYED: + (preexisting if base_verdict == DECAYED else new_decays).append(f) + elif base_verdict == DECAYED: + fixed.append(f) + return new_decays, preexisting, fixed, None + + +def run_since(base: str, root: str) -> int: + new_decays, preexisting, fixed, err = since_regression(base, root) + if err is not None: + print(f"citation-decay: ERROR: {err}", file=sys.stderr) + return 1 + print( + f"citation-decay --since {base}: " + f"{len(new_decays)} new decay(s), {len(preexisting)} pre-existing " + f"(backlog, not failing), {len(fixed)} fixed since base" + ) + if new_decays: + print("\nNEW DECAY (introduced or exposed since base):") + for f in new_decays[:MAX_EXAMPLES]: + print(" " + str(f)) + if len(new_decays) > MAX_EXAMPLES: + print(f" ... and {len(new_decays) - MAX_EXAMPLES} more") + if preexisting: + print(f"\npre-existing backlog (first {min(MAX_EXAMPLES, len(preexisting))}, not failing):") + for f in preexisting[:MAX_EXAMPLES]: + print(" " + str(f)) + if fixed: + print(f"\nfixed since base ({len(fixed)}):") + for f in fixed[:MAX_EXAMPLES]: + print(" " + str(f)) + if new_decays: + print(FIX_MESSAGE) + return 1 + print("\nno new citation decay since base") + return 0 + + +def run(paths: list[str], root: str) -> int: findings: list[Finding] = [] for p in sorted(set(paths)): if os.path.isfile(p): - got = scan_file(p, root) - if only is not None: - keep = only.get(os.path.relpath(p, root).replace(os.sep, "/"), set()) - got = [f for f in got if f.cite_line in keep] - findings.extend(got) + findings.extend(scan_file(p, root)) counts = {OK: 0, DECAYED: 0, UNVERIFIABLE: 0} for f in findings: counts[f.verdict] += 1 @@ -380,27 +495,123 @@ def self_test() -> int: if mark == "FAIL": failures.append(key) print("--- self-test " + ("FAILED ---" if failures else "PASSED ---")) - return 1 if failures else 0 + + regression_failures = self_test_since_regression() + return 1 if (failures or regression_failures) else 0 + + +def _git(args: list[str], cwd: str) -> None: + subprocess.run( + ["git"] + args, cwd=cwd, capture_output=True, text=True, check=True, + env={**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", + "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t"}, + ) + + +def self_test_since_regression() -> int: + """Prove `--since` fires on a NEW decay and stays silent on backlog. + + This is the falsifier for the defect that motivated `--since` in the + first place: `--added-lines-only` filters by which lines a PR ADDS, but + a citation decays when its TARGET file changes on lines the PR never + touches (`EPIPHANIES.md` prepend) -- so the old mode reported success on + precisely the change it exists to catch. Both halves below are real git + history across two commits, not a synthetic verdict comparison. + + CAN-FIRE: file A cites `B.md:8` correctly at base. Head PREPENDS lines to + B.md (A is untouched). A NEW decay must be reported and exit 1. + + MUST-STAY-SILENT: in the same repo, a citation already DECAYED at base + and unchanged at head must NOT be reported as new, and must not by + itself cause a nonzero exit. + """ + d = tempfile.mkdtemp(prefix="citation-decay-selftest-since-") + sub = os.path.join(d, ".claude", "board") + os.makedirs(sub) + b_path = os.path.join(sub, "B.md") + a_path = os.path.join(sub, "A.md") + + def write_b(lines: list[str]) -> None: + with open(b_path, "w") as fh: + fh.write("\n".join(lines) + "\n") + + # base B.md: the CAN-FIRE anchor's real home sits exactly at its cited + # line 8 (OK at base). The MUST-STAY-SILENT anchor is cited at line 3 + # but its real home is line 20 -- already outside the +-WINDOW_LINES(3) + # window at base, i.e. ALREADY DECAYED at base -- pre-existing backlog. + base_lines = [f"line {i} filler" for i in range(1, 22)] + base_lines[7] = "## E-CANFIRE-ANCHOR-1 heading" # line 8 (0-indexed 7) + base_lines[2] = "line 3 filler (backlog cite target)" # line 3 + base_lines[19] = "## E-BACKLOG-ANCHOR-1 heading" # line 20 + write_b(base_lines) + with open(a_path, "w") as fh: + fh.write( + "The ruling E-CANFIRE-ANCHOR-1 is recorded at .claude/board/B.md:8.\n" + + "prose padding, no anchor here at all whatsoever indeed. " * 8 + "\n" + "The ruling E-BACKLOG-ANCHOR-1 is recorded at .claude/board/B.md:3.\n" + + "more prose padding, still nothing citable in it at all. " * 8 + "\n" + ) + _git(["init", "-q"], d) + _git(["add", "-A"], d) + _git(["commit", "-q", "-m", "base"], d) + + # head: PREPEND 5 lines to B.md (> WINDOW_LINES so the shift cannot hide + # inside the anchor-search window). A.md is untouched -- the citing + # lines themselves never move. This pushes the CAN-FIRE anchor's real + # home from line 8 to line 13, outside cited-line-8's +-3 window: a real + # NEW decay on an unchanged citing line. The backlog anchor shifts from + # 20 to 25, still outside cited-line-3's +-3 window either side of the + # prepend, so it stays exactly as decayed as it was at base. + head_lines = ["PREPENDED filler"] * 5 + base_lines + write_b(head_lines) + _git(["add", "-A"], d) + _git(["commit", "-q", "-m", "prepend to B.md, A.md untouched"], d) + + new_decays, preexisting, fixed, err = since_regression("HEAD~1", d) + print("\n--- self-test --since findings ---") + if err is not None: + print(f" ERROR: {err}") + print("--- self-test --since FAILED ---") + return 1 + for label, group in (("new", new_decays), ("preexisting", preexisting), ("fixed", fixed)): + for f in group: + print(f" [{label}] " + str(f)) + + fail_msgs = [] + canfire_new = any(f.cited == ".claude/board/B.md" and f.start == 8 for f in new_decays) + if not canfire_new: + fail_msgs.append("CAN-FIRE: .claude/board/B.md:8 must be reported as a NEW decay") + backlog_new = any(f.cited == ".claude/board/B.md" and f.start == 3 for f in new_decays) + if backlog_new: + fail_msgs.append("MUST-STAY-SILENT: .claude/board/B.md:3 (pre-existing decay) must NOT be reported as new") + backlog_preexisting = any(f.cited == ".claude/board/B.md" and f.start == 3 for f in preexisting) + if not backlog_preexisting: + fail_msgs.append("MUST-STAY-SILENT: .claude/board/B.md:3 must be counted as pre-existing backlog") + + # The verdict the GATE would return on this corpus -- NOT the self-test's + # own pass/fail. The first version printed `0 if ok else 1` under a label + # reading "expect 1", so a PASSING run printed `0` next to the word + # "expect 1" and asserted nothing whatsoever about the real exit path. + gate_rc = 1 if new_decays else 0 + if gate_rc != 1: + fail_msgs.append("CAN-FIRE: the gate must exit 1 while a new decay is present") + + for m in fail_msgs: + print(f" [FAIL] {m}") + ok = not fail_msgs + print(f" gate exit code on this corpus: {gate_rc} (expect 1 -- new decay present)") + print("--- self-test --since " + ("PASSED ---" if ok else "FAILED ---")) + return 0 if ok else 1 def main(argv: list[str]) -> int: root = os.getcwd() if "--self-test" in argv: return self_test() - only = None - if "--added-lines-only" in argv: - i = argv.index("--added-lines-only") + if "--since" in argv: + i = argv.index("--since") base = argv[i + 1] - only = added_lines(base, root) - if only is None: - print( - f"citation-decay: ERROR: cannot diff '{base}...HEAD' -- refusing to " - "check every line instead (in CI this usually means a shallow " - "checkout; the workflow needs `fetch-depth: 0`).", - file=sys.stderr, - ) - return 1 - argv = argv[:i] + argv[i + 2:] + return run_since(base, root) pats = [a for a in argv if not a.startswith("-")] or DEFAULT_GLOBS paths: list[str] = [] for p in pats: @@ -409,7 +620,7 @@ def main(argv: list[str]) -> int: if not paths: print("citation-decay: no input files matched", file=sys.stderr) return 0 - return run(paths, root, only) + return run(paths, root) if __name__ == "__main__": diff --git a/.github/workflows/citation-decay.yml b/.github/workflows/citation-decay.yml index cc93d98dd..d9c7464d5 100644 --- a/.github/workflows/citation-decay.yml +++ b/.github/workflows/citation-decay.yml @@ -2,10 +2,11 @@ name: Citations have not decayed on: pull_request: paths: - # The gate's inputs are the CITING files -- a plan or board file is what - # carries a `path:LINE` reference. `.claude/board/EPIPHANIES.md` is the - # motivating case: it is APPEND-ONLY and PREPENDED to, so every line - # number into it shifts on every new entry, with no edit on either side. + # The gate's inputs are BOTH the citing files and the cited ones -- a + # prepend to `.claude/board/EPIPHANIES.md` moves targets under citations + # that live in other, unchanged files on unchanged lines. That is the + # motivating case, and it is why this runs on the whole board/plan tree + # rather than on the diff. - .claude/plans/** - .claude/board/** # ...and the gate itself. @@ -25,40 +26,32 @@ jobs: steps: - uses: actions/checkout@v4 with: - # The gate compares this PR's citing files against the merge base, so - # it needs both sides of the history, not a depth-1 tip. + # The gate builds a worktree at the merge base and compares verdicts + # against it, so it needs both sides of the history, not a depth-1 tip. fetch-depth: 0 - name: Self-test the gate # A gate whose own falsifiers are broken cannot be trusted to report on - # anything else. Both halves are asserted: it FIRES on a moved anchor - # and STAYS SILENT on a correct one and on an unverifiable one. + # anything else. Both halves are asserted, in both the single-revision + # and the two-revision arm: it FIRES on a decay this PR introduced or + # exposed, and STAYS SILENT on the pre-existing backlog. run: python3 .claude/tools/citation_decay.py --self-test - - name: Check citations in this PR's plan/board files - run: | - set -euo pipefail - BASE="${{ github.event.pull_request.base.sha }}" - # SCOPED TO CHANGED *LINES*, not changed files. A repo-wide run reports - # pre-existing backlog (measured 2026-09-04: 124 confirmed decays - # across .claude/plans + .claude/board), so a repo-wide gate would - # fail every PR regardless of what it changed -- and a gate that - # fires on everything carries exactly as much information as one that - # never fires. - # - # File-level scoping is NOT enough and was measured so: EPIPHANIES.md - # alone carries 10 pre-existing decays, and this workspace's - # board-hygiene rule means nearly every PR touches it. `--added-lines-only` - # restricts the verdict to lines this PR actually added or modified, - # which is what makes it a NO-NEW-DECAY gate. The backlog is a - # separate, deliberate cleanup. - FILES=$(git diff --name-only --diff-filter=d "$BASE"...HEAD \ - -- '.claude/plans/*.md' '.claude/board/*.md' || true) - if [ -z "$FILES" ]; then - echo "no plan/board files changed in this PR; nothing to check" - exit 0 - fi - echo "checking:" - echo "$FILES" | sed 's/^/ /' - # shellcheck disable=SC2086 - python3 .claude/tools/citation_decay.py --added-lines-only "$BASE" $FILES + - name: Check for citation decay introduced by this PR + # A REGRESSION comparison, not a diff filter. `--since` scans the whole + # corpus at the merge base and at HEAD and fails ONLY where a citation + # is decayed now and was not decayed at the base. + # + # The first version of this step filtered to the lines a PR ADDED, and + # was structurally blind to its own motivating case: prepending to + # EPIPHANIES.md decays citations in files the PR never touched. It also + # computed a file list with `|| true`, so an unresolvable merge base + # became an empty list and a GREEN run -- bypassing the script's own + # fail-closed handling. Both are gone: there is no file list to compute, + # and the script fails closed on any git failure. + # + # The pre-existing backlog (measured 148 decays on 2026-09-04, up from + # 124 as the corpus grew) is reported and never fails the job. A gate + # that fires on everything carries exactly as much information as one + # that never fires. + run: python3 .claude/tools/citation_decay.py --since "${{ github.event.pull_request.base.sha }}"