From 3ca9b8e88e561409a7dcce635f024b99d172a3a1 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:09:29 +0800 Subject: [PATCH 1/3] Add safe human handoff for source workflows Co-Authored-By: Codex --- .github/workflows/rss_source_pipeline.yml | 11 ++- .github/workflows/source_event_pipeline.yml | 11 ++- scripts/publish_live_outputs_pr.py | 74 ++++++++++++++++++--- tests/test_publish_live_outputs_pr.py | 69 +++++++++++++++++++ 4 files changed, 145 insertions(+), 20 deletions(-) diff --git a/.github/workflows/rss_source_pipeline.yml b/.github/workflows/rss_source_pipeline.yml index 83d541c..eb0da7a 100644 --- a/.github/workflows/rss_source_pipeline.yml +++ b/.github/workflows/rss_source_pipeline.yml @@ -35,8 +35,7 @@ on: - cron: "15 12 * * 6" permissions: - contents: write - pull-requests: write + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -100,16 +99,16 @@ jobs: data/live/political_events.csv \ data/live/source_tracker.csv git add data/live/source_items.csv data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_fetch_status.json data/live/source_manifest.json - - name: Create or update protected live-output PR - env: - GH_TOKEN: ${{ github.token }} + - name: Create human-review publication handoff run: | python scripts/publish_live_outputs_pr.py \ --branch automation/live-rss-source-events \ --title "Update generated live RSS source events" \ --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ - --commit-message "Update generated live RSS source events" + --commit-message "Update generated live RSS source events" \ + --handoff-dir data/output/rss_source_pipeline - name: Upload RSS source artifact + if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: rss-source-pipeline diff --git a/.github/workflows/source_event_pipeline.yml b/.github/workflows/source_event_pipeline.yml index 60d9457..7839f12 100644 --- a/.github/workflows/source_event_pipeline.yml +++ b/.github/workflows/source_event_pipeline.yml @@ -30,8 +30,7 @@ on: - cron: "45 12 * * 6" permissions: - contents: write - pull-requests: write + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -91,16 +90,16 @@ jobs: --output data/live/source_manifest.json \ "${MANIFEST_PATHS[@]}" git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json - - name: Create or update protected live-output PR - env: - GH_TOKEN: ${{ github.token }} + - name: Create human-review publication handoff run: | python scripts/publish_live_outputs_pr.py \ --branch automation/live-source-events \ --title "Update generated live source events" \ --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ - --commit-message "Update generated live source events" + --commit-message "Update generated live source events" \ + --handoff-dir data/output/source_event_pipeline - name: Upload source event artifact + if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: source-event-pipeline diff --git a/scripts/publish_live_outputs_pr.py b/scripts/publish_live_outputs_pr.py index 76fc3ec..96d81ad 100644 --- a/scripts/publish_live_outputs_pr.py +++ b/scripts/publish_live_outputs_pr.py @@ -1,25 +1,28 @@ #!/usr/bin/env python3 -"""Publish generated research data through a protected-branch pull request. +"""Prepare generated research data for protected-branch publication. Scheduled workflows must not push generated data directly to ``main``. This -small, repository-local adapter commits only the paths staged by its caller to -an automation branch and creates (or reuses) a pull request for the normal CI -and branch-protection path. +repository-local adapter can create an auditable patch and HUMAN_REQUIRED +receipt without mutating the remote. Its legacy PR path remains available for +an explicitly approved identity. """ from __future__ import annotations import argparse +import hashlib +import json import os import subprocess from collections.abc import Sequence +from pathlib import Path class PublishError(RuntimeError): """Raised when a generated-data PR cannot be created safely.""" -def run(command: Sequence[str], *, capture: bool = False) -> str: +def run(command: Sequence[str], *, capture: bool = False, strip: bool = True) -> str: completed = subprocess.run( command, check=False, @@ -30,7 +33,8 @@ def run(command: Sequence[str], *, capture: bool = False) -> str: if completed.returncode: detail = (completed.stderr or completed.stdout or "command failed").strip() raise PublishError(f"{' '.join(command)}: {detail}") - return (completed.stdout or "").strip() + output = completed.stdout or "" + return output.strip() if strip else output def staged_changes_present() -> bool: @@ -58,10 +62,55 @@ def existing_pull_request(branch: str) -> str: ) -def publish(branch: str, title: str, body: str, commit_message: str) -> str: +def write_handoff(handoff_dir: Path, branch: str, title: str) -> None: + handoff_dir.mkdir(parents=True, exist_ok=True) + patch_path = handoff_dir / "generated-live-output.patch" + patch = run( + ["git", "diff", "--binary", "--full-index", "--cached"], + capture=True, + strip=False, + ) + patch_path.write_text(patch, encoding="utf-8") + staged_paths = run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], + capture=True, + ).splitlines() + receipt = { + "schema_version": 1, + "status": "HUMAN_REQUIRED", + "reason_code": "PR_CREATION_IDENTITY_UNAVAILABLE", + "repository": os.environ.get("GITHUB_REPOSITORY", ""), + "source_sha": os.environ.get("GITHUB_SHA", ""), + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "target_branch": "main", + "proposed_branch": branch, + "title": title, + "staged_paths": staged_paths, + "patch_path": patch_path.name, + "patch_sha256": hashlib.sha256(patch_path.read_bytes()).hexdigest(), + "next_action": "A maintainer must review and apply the patch through the protected branch process.", + } + (handoff_dir / "publication-handoff.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def publish( + branch: str, + title: str, + body: str, + commit_message: str, + *, + handoff_dir: Path | None = None, +) -> str: if not staged_changes_present(): return "No generated data changes to publish." + if handoff_dir is not None: + write_handoff(handoff_dir, branch, title) + return "HUMAN_REQUIRED: generated publication handoff artifact." + run(["git", "config", "user.name", "github-actions[bot]"]) run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) run(["git", "switch", "-C", branch]) @@ -99,12 +148,21 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--title", required=True) parser.add_argument("--body", required=True) parser.add_argument("--commit-message", required=True) + parser.add_argument("--handoff-dir", type=Path) return parser.parse_args() def main() -> None: args = parse_args() - print(publish(args.branch, args.title, args.body, args.commit_message)) + print( + publish( + args.branch, + args.title, + args.body, + args.commit_message, + handoff_dir=args.handoff_dir, + ) + ) if __name__ == "__main__": diff --git a/tests/test_publish_live_outputs_pr.py b/tests/test_publish_live_outputs_pr.py index f055e11..64ca7aa 100644 --- a/tests/test_publish_live_outputs_pr.py +++ b/tests/test_publish_live_outputs_pr.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import json from pathlib import Path @@ -59,3 +60,71 @@ def fake_run(command, *, capture=False): create = next(command for command in commands if command[:3] == ["gh", "pr", "create"]) assert ["--head", "automation/generated"] == create[create.index("--head") : create.index("--head") + 2] assert ["--base", "main"] == create[create.index("--base") : create.index("--base") + 2] + + +def test_publish_writes_human_handoff_without_remote_mutation(monkeypatch, tmp_path: Path) -> None: + module = load_module() + commands: list[list[str]] = [] + monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "123") + monkeypatch.setattr(module, "staged_changes_present", lambda: True) + + def fake_run(command, *, capture=False, strip=True): + commands.append(list(command)) + if command[-3:] == ["--binary", "--full-index", "--cached"]: + assert strip is False + return "diff --git a/data/live/example.csv b/data/live/example.csv\n" + if command[-3:] == ["--cached", "--name-only", "--diff-filter=ACMR"]: + return "data/live/example.csv" + return "" + + monkeypatch.setattr(module, "run", fake_run) + + result = module.publish( + "automation/generated", + "Generated", + "body", + "commit", + handoff_dir=tmp_path, + ) + + assert result == "HUMAN_REQUIRED: generated publication handoff artifact." + assert not any(command[:2] == ["git", "push"] for command in commands) + assert not any(command[:3] == ["gh", "pr", "create"] for command in commands) + + patch = tmp_path / "generated-live-output.patch" + receipt = json.loads((tmp_path / "publication-handoff.json").read_text(encoding="utf-8")) + assert patch.read_text(encoding="utf-8").startswith("diff --git") + assert patch.read_bytes().endswith(b"\n") + assert receipt == { + "schema_version": 1, + "status": "HUMAN_REQUIRED", + "reason_code": "PR_CREATION_IDENTITY_UNAVAILABLE", + "repository": "QuantStrategyLab/example", + "source_sha": "a" * 40, + "run_id": "123", + "target_branch": "main", + "proposed_branch": "automation/generated", + "title": "Generated", + "staged_paths": ["data/live/example.csv"], + "patch_path": "generated-live-output.patch", + "patch_sha256": module.hashlib.sha256(patch.read_bytes()).hexdigest(), + "next_action": "A maintainer must review and apply the patch through the protected branch process.", + } + + +def test_source_workflows_use_read_only_handoff_artifacts() -> None: + root = Path(__file__).parents[1] + workflows = { + "rss_source_pipeline.yml": "data/output/rss_source_pipeline", + "source_event_pipeline.yml": "data/output/source_event_pipeline", + } + + for filename, output_dir in workflows.items(): + text = (root / ".github" / "workflows" / filename).read_text(encoding="utf-8") + assert "permissions:\n contents: read" in text + assert "contents: write" not in text + assert "pull-requests: write" not in text + assert f"--handoff-dir {output_dir}" in text + assert "if: ${{ always() }}" in text From 813c6ab414bae5b1c91e5b65d24bc845ad837d7f Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:14:47 +0800 Subject: [PATCH 2/3] fix: require publication handoff Co-Authored-By: Codex --- scripts/publish_live_outputs_pr.py | 65 +++------------------------ tests/test_publish_live_outputs_pr.py | 47 +++++++------------ 2 files changed, 24 insertions(+), 88 deletions(-) diff --git a/scripts/publish_live_outputs_pr.py b/scripts/publish_live_outputs_pr.py index 96d81ad..2ab54c4 100644 --- a/scripts/publish_live_outputs_pr.py +++ b/scripts/publish_live_outputs_pr.py @@ -1,10 +1,8 @@ -#!/usr/bin/env python3 """Prepare generated research data for protected-branch publication. Scheduled workflows must not push generated data directly to ``main``. This repository-local adapter can create an auditable patch and HUMAN_REQUIRED -receipt without mutating the remote. Its legacy PR path remains available for -an explicitly approved identity. +receipt without mutating the repository or remote. """ from __future__ import annotations @@ -41,27 +39,6 @@ def staged_changes_present() -> bool: return subprocess.run(["git", "diff", "--cached", "--quiet"], check=False).returncode != 0 -def existing_pull_request(branch: str) -> str: - return run( - [ - "gh", - "pr", - "list", - "--repo", - os.environ["GITHUB_REPOSITORY"], - "--state", - "open", - "--head", - branch, - "--json", - "url", - "--jq", - ".[0].url // \"\"", - ], - capture=True, - ) - - def write_handoff(handoff_dir: Path, branch: str, title: str) -> None: handoff_dir.mkdir(parents=True, exist_ok=True) patch_path = handoff_dir / "generated-live-output.patch" @@ -104,42 +81,14 @@ def publish( *, handoff_dir: Path | None = None, ) -> str: + if handoff_dir is None: + raise PublishError("--handoff-dir is required; remote publication is disabled") + if not staged_changes_present(): return "No generated data changes to publish." - if handoff_dir is not None: - write_handoff(handoff_dir, branch, title) - return "HUMAN_REQUIRED: generated publication handoff artifact." - - run(["git", "config", "user.name", "github-actions[bot]"]) - run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) - run(["git", "switch", "-C", branch]) - run(["git", "commit", "-m", commit_message]) - run(["git", "push", "--force-with-lease", "origin", f"HEAD:refs/heads/{branch}"]) - - existing_url = existing_pull_request(branch) - if existing_url: - return f"Updated generated-data PR: {existing_url}" - - url = run( - [ - "gh", - "pr", - "create", - "--repo", - os.environ["GITHUB_REPOSITORY"], - "--head", - branch, - "--base", - "main", - "--title", - title, - "--body", - body, - ], - capture=True, - ) - return f"Created generated-data PR: {url}" + write_handoff(handoff_dir, branch, title) + return "HUMAN_REQUIRED: generated publication handoff artifact." def parse_args() -> argparse.Namespace: @@ -148,7 +97,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--title", required=True) parser.add_argument("--body", required=True) parser.add_argument("--commit-message", required=True) - parser.add_argument("--handoff-dir", type=Path) + parser.add_argument("--handoff-dir", type=Path, required=True) return parser.parse_args() diff --git a/tests/test_publish_live_outputs_pr.py b/tests/test_publish_live_outputs_pr.py index 64ca7aa..2c2b846 100644 --- a/tests/test_publish_live_outputs_pr.py +++ b/tests/test_publish_live_outputs_pr.py @@ -4,6 +4,8 @@ import json from pathlib import Path +import pytest + def load_module(): path = Path(__file__).parents[1] / "scripts" / "publish_live_outputs_pr.py" @@ -14,52 +16,37 @@ def load_module(): return module -def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch) -> None: +def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch, tmp_path: Path) -> None: module = load_module() monkeypatch.setattr(module, "staged_changes_present", lambda: False) - assert module.publish("automation/generated", "Generated", "body", "commit") == "No generated data changes to publish." - - -def test_publish_uses_an_automation_branch_and_reuses_existing_pr(monkeypatch) -> None: - module = load_module() - commands: list[list[str]] = [] - monkeypatch.setattr(module, "staged_changes_present", lambda: True) - monkeypatch.setattr(module, "existing_pull_request", lambda branch: "https://example.test/pr/1") - - def fake_run(command, *, capture=False): - commands.append(list(command)) - return "" - - monkeypatch.setattr(module, "run", fake_run) - - result = module.publish("automation/generated", "Generated", "body", "commit") + result = module.publish( + "automation/generated", + "Generated", + "body", + "commit", + handoff_dir=tmp_path, + ) - assert result == "Updated generated-data PR: https://example.test/pr/1" - assert ["git", "switch", "-C", "automation/generated"] in commands - assert ["git", "push", "--force-with-lease", "origin", "HEAD:refs/heads/automation/generated"] in commands - assert all("main" not in command[-1:] for command in commands if command[:2] == ["git", "push"]) + assert result == "No generated data changes to publish." -def test_publish_creates_pull_request_after_pushing_branch(monkeypatch) -> None: +def test_publish_requires_handoff_before_repository_or_remote_mutation(monkeypatch) -> None: module = load_module() commands: list[list[str]] = [] monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") monkeypatch.setattr(module, "staged_changes_present", lambda: True) - monkeypatch.setattr(module, "existing_pull_request", lambda branch: "") - def fake_run(command, *, capture=False): + def fake_run(command, *, capture=False, strip=True): commands.append(list(command)) - return "https://example.test/pr/2" if command[:3] == ["gh", "pr", "create"] else "" + return "" monkeypatch.setattr(module, "run", fake_run) - result = module.publish("automation/generated", "Generated", "body", "commit") + with pytest.raises(module.PublishError, match="handoff"): + module.publish("automation/generated", "Generated", "body", "commit") - assert result == "Created generated-data PR: https://example.test/pr/2" - create = next(command for command in commands if command[:3] == ["gh", "pr", "create"]) - assert ["--head", "automation/generated"] == create[create.index("--head") : create.index("--head") + 2] - assert ["--base", "main"] == create[create.index("--base") : create.index("--base") + 2] + assert commands == [] def test_publish_writes_human_handoff_without_remote_mutation(monkeypatch, tmp_path: Path) -> None: From 043bf3f88e8a4fdd3767a45409d9693d80b945cb Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 2 Sep 2026 05:28:59 +0800 Subject: [PATCH 3/3] test: lock down publication handoff boundary Co-Authored-By: Codex --- tests/test_publish_live_outputs_pr.py | 39 +++++++++++++++++++-------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/tests/test_publish_live_outputs_pr.py b/tests/test_publish_live_outputs_pr.py index 2c2b846..86f5c69 100644 --- a/tests/test_publish_live_outputs_pr.py +++ b/tests/test_publish_live_outputs_pr.py @@ -2,6 +2,7 @@ import importlib.util import json +import sys from pathlib import Path import pytest @@ -33,20 +34,34 @@ def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch def test_publish_requires_handoff_before_repository_or_remote_mutation(monkeypatch) -> None: module = load_module() - commands: list[list[str]] = [] monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") - monkeypatch.setattr(module, "staged_changes_present", lambda: True) - - def fake_run(command, *, capture=False, strip=True): - commands.append(list(command)) - return "" - - monkeypatch.setattr(module, "run", fake_run) + monkeypatch.setattr(module, "staged_changes_present", lambda: pytest.fail("git diff must not run")) + monkeypatch.setattr(module, "run", lambda *_args, **_kwargs: pytest.fail("git command must not run")) with pytest.raises(module.PublishError, match="handoff"): module.publish("automation/generated", "Generated", "body", "commit") - assert commands == [] + +def test_cli_requires_handoff_dir(monkeypatch) -> None: + module = load_module() + monkeypatch.setattr( + sys, + "argv", + [ + "publish_live_outputs_pr.py", + "--branch", + "automation/generated", + "--title", + "Generated", + "--body", + "body", + "--commit-message", + "commit", + ], + ) + + with pytest.raises(SystemExit): + module.parse_args() def test_publish_writes_human_handoff_without_remote_mutation(monkeypatch, tmp_path: Path) -> None: @@ -77,8 +92,10 @@ def fake_run(command, *, capture=False, strip=True): ) assert result == "HUMAN_REQUIRED: generated publication handoff artifact." - assert not any(command[:2] == ["git", "push"] for command in commands) - assert not any(command[:3] == ["gh", "pr", "create"] for command in commands) + assert commands == [ + ["git", "diff", "--binary", "--full-index", "--cached"], + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], + ] patch = tmp_path / "generated-live-output.patch" receipt = json.loads((tmp_path / "publication-handoff.json").read_text(encoding="utf-8"))