diff --git a/.github/workflows/rss_source_pipeline.yml b/.github/workflows/rss_source_pipeline.yml index 83d541c..eb0da7a 100644 --- a/.github/workflows/rss_source_pipeline.yml +++ b/.github/workflows/rss_source_pipeline.yml @@ -35,8 +35,7 @@ on: - cron: "15 12 * * 6" permissions: - contents: write - pull-requests: write + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -100,16 +99,16 @@ jobs: data/live/political_events.csv \ data/live/source_tracker.csv git add data/live/source_items.csv data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_fetch_status.json data/live/source_manifest.json - - name: Create or update protected live-output PR - env: - GH_TOKEN: ${{ github.token }} + - name: Create human-review publication handoff run: | python scripts/publish_live_outputs_pr.py \ --branch automation/live-rss-source-events \ --title "Update generated live RSS source events" \ --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ - --commit-message "Update generated live RSS source events" + --commit-message "Update generated live RSS source events" \ + --handoff-dir data/output/rss_source_pipeline - name: Upload RSS source artifact + if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: rss-source-pipeline diff --git a/.github/workflows/source_event_pipeline.yml b/.github/workflows/source_event_pipeline.yml index 60d9457..7839f12 100644 --- a/.github/workflows/source_event_pipeline.yml +++ b/.github/workflows/source_event_pipeline.yml @@ -30,8 +30,7 @@ on: - cron: "45 12 * * 6" permissions: - contents: write - pull-requests: write + contents: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name }} @@ -91,16 +90,16 @@ jobs: --output data/live/source_manifest.json \ "${MANIFEST_PATHS[@]}" git add data/live/source_events.csv data/live/political_events.csv data/live/source_tracker.csv data/live/source_manifest.json - - name: Create or update protected live-output PR - env: - GH_TOKEN: ${{ github.token }} + - name: Create human-review publication handoff run: | python scripts/publish_live_outputs_pr.py \ --branch automation/live-source-events \ --title "Update generated live source events" \ --body "Automated research-data update. Generated files are reviewed through the normal protected-branch CI path; this workflow never pushes to main directly." \ - --commit-message "Update generated live source events" + --commit-message "Update generated live source events" \ + --handoff-dir data/output/source_event_pipeline - name: Upload source event artifact + if: ${{ always() }} uses: actions/upload-artifact@v7 with: name: source-event-pipeline diff --git a/scripts/publish_live_outputs_pr.py b/scripts/publish_live_outputs_pr.py index 76fc3ec..2ab54c4 100644 --- a/scripts/publish_live_outputs_pr.py +++ b/scripts/publish_live_outputs_pr.py @@ -1,25 +1,26 @@ -#!/usr/bin/env python3 -"""Publish generated research data through a protected-branch pull request. +"""Prepare generated research data for protected-branch publication. Scheduled workflows must not push generated data directly to ``main``. This -small, repository-local adapter commits only the paths staged by its caller to -an automation branch and creates (or reuses) a pull request for the normal CI -and branch-protection path. +repository-local adapter can create an auditable patch and HUMAN_REQUIRED +receipt without mutating the repository or remote. """ from __future__ import annotations import argparse +import hashlib +import json import os import subprocess from collections.abc import Sequence +from pathlib import Path class PublishError(RuntimeError): """Raised when a generated-data PR cannot be created safely.""" -def run(command: Sequence[str], *, capture: bool = False) -> str: +def run(command: Sequence[str], *, capture: bool = False, strip: bool = True) -> str: completed = subprocess.run( command, check=False, @@ -30,67 +31,64 @@ def run(command: Sequence[str], *, capture: bool = False) -> str: if completed.returncode: detail = (completed.stderr or completed.stdout or "command failed").strip() raise PublishError(f"{' '.join(command)}: {detail}") - return (completed.stdout or "").strip() + output = completed.stdout or "" + return output.strip() if strip else output def staged_changes_present() -> bool: return subprocess.run(["git", "diff", "--cached", "--quiet"], check=False).returncode != 0 -def existing_pull_request(branch: str) -> str: - return run( - [ - "gh", - "pr", - "list", - "--repo", - os.environ["GITHUB_REPOSITORY"], - "--state", - "open", - "--head", - branch, - "--json", - "url", - "--jq", - ".[0].url // \"\"", - ], +def write_handoff(handoff_dir: Path, branch: str, title: str) -> None: + handoff_dir.mkdir(parents=True, exist_ok=True) + patch_path = handoff_dir / "generated-live-output.patch" + patch = run( + ["git", "diff", "--binary", "--full-index", "--cached"], capture=True, + strip=False, ) + patch_path.write_text(patch, encoding="utf-8") + staged_paths = run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], + capture=True, + ).splitlines() + receipt = { + "schema_version": 1, + "status": "HUMAN_REQUIRED", + "reason_code": "PR_CREATION_IDENTITY_UNAVAILABLE", + "repository": os.environ.get("GITHUB_REPOSITORY", ""), + "source_sha": os.environ.get("GITHUB_SHA", ""), + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "target_branch": "main", + "proposed_branch": branch, + "title": title, + "staged_paths": staged_paths, + "patch_path": patch_path.name, + "patch_sha256": hashlib.sha256(patch_path.read_bytes()).hexdigest(), + "next_action": "A maintainer must review and apply the patch through the protected branch process.", + } + (handoff_dir / "publication-handoff.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + +def publish( + branch: str, + title: str, + body: str, + commit_message: str, + *, + handoff_dir: Path | None = None, +) -> str: + if handoff_dir is None: + raise PublishError("--handoff-dir is required; remote publication is disabled") -def publish(branch: str, title: str, body: str, commit_message: str) -> str: if not staged_changes_present(): return "No generated data changes to publish." - run(["git", "config", "user.name", "github-actions[bot]"]) - run(["git", "config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) - run(["git", "switch", "-C", branch]) - run(["git", "commit", "-m", commit_message]) - run(["git", "push", "--force-with-lease", "origin", f"HEAD:refs/heads/{branch}"]) - - existing_url = existing_pull_request(branch) - if existing_url: - return f"Updated generated-data PR: {existing_url}" - - url = run( - [ - "gh", - "pr", - "create", - "--repo", - os.environ["GITHUB_REPOSITORY"], - "--head", - branch, - "--base", - "main", - "--title", - title, - "--body", - body, - ], - capture=True, - ) - return f"Created generated-data PR: {url}" + write_handoff(handoff_dir, branch, title) + return "HUMAN_REQUIRED: generated publication handoff artifact." def parse_args() -> argparse.Namespace: @@ -99,12 +97,21 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--title", required=True) parser.add_argument("--body", required=True) parser.add_argument("--commit-message", required=True) + parser.add_argument("--handoff-dir", type=Path, required=True) return parser.parse_args() def main() -> None: args = parse_args() - print(publish(args.branch, args.title, args.body, args.commit_message)) + print( + publish( + args.branch, + args.title, + args.body, + args.commit_message, + handoff_dir=args.handoff_dir, + ) + ) if __name__ == "__main__": diff --git a/tests/test_publish_live_outputs_pr.py b/tests/test_publish_live_outputs_pr.py index f055e11..86f5c69 100644 --- a/tests/test_publish_live_outputs_pr.py +++ b/tests/test_publish_live_outputs_pr.py @@ -1,8 +1,12 @@ from __future__ import annotations import importlib.util +import json +import sys from pathlib import Path +import pytest + def load_module(): path = Path(__file__).parents[1] / "scripts" / "publish_live_outputs_pr.py" @@ -13,49 +17,118 @@ def load_module(): return module -def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch) -> None: +def test_publish_does_not_create_branch_or_pr_without_staged_changes(monkeypatch, tmp_path: Path) -> None: module = load_module() monkeypatch.setattr(module, "staged_changes_present", lambda: False) - assert module.publish("automation/generated", "Generated", "body", "commit") == "No generated data changes to publish." - + result = module.publish( + "automation/generated", + "Generated", + "body", + "commit", + handoff_dir=tmp_path, + ) -def test_publish_uses_an_automation_branch_and_reuses_existing_pr(monkeypatch) -> None: - module = load_module() - commands: list[list[str]] = [] - monkeypatch.setattr(module, "staged_changes_present", lambda: True) - monkeypatch.setattr(module, "existing_pull_request", lambda branch: "https://example.test/pr/1") - - def fake_run(command, *, capture=False): - commands.append(list(command)) - return "" + assert result == "No generated data changes to publish." - monkeypatch.setattr(module, "run", fake_run) - result = module.publish("automation/generated", "Generated", "body", "commit") +def test_publish_requires_handoff_before_repository_or_remote_mutation(monkeypatch) -> None: + module = load_module() + monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") + monkeypatch.setattr(module, "staged_changes_present", lambda: pytest.fail("git diff must not run")) + monkeypatch.setattr(module, "run", lambda *_args, **_kwargs: pytest.fail("git command must not run")) - assert result == "Updated generated-data PR: https://example.test/pr/1" - assert ["git", "switch", "-C", "automation/generated"] in commands - assert ["git", "push", "--force-with-lease", "origin", "HEAD:refs/heads/automation/generated"] in commands - assert all("main" not in command[-1:] for command in commands if command[:2] == ["git", "push"]) + with pytest.raises(module.PublishError, match="handoff"): + module.publish("automation/generated", "Generated", "body", "commit") -def test_publish_creates_pull_request_after_pushing_branch(monkeypatch) -> None: +def test_cli_requires_handoff_dir(monkeypatch) -> None: + module = load_module() + monkeypatch.setattr( + sys, + "argv", + [ + "publish_live_outputs_pr.py", + "--branch", + "automation/generated", + "--title", + "Generated", + "--body", + "body", + "--commit-message", + "commit", + ], + ) + + with pytest.raises(SystemExit): + module.parse_args() + + +def test_publish_writes_human_handoff_without_remote_mutation(monkeypatch, tmp_path: Path) -> None: module = load_module() commands: list[list[str]] = [] monkeypatch.setenv("GITHUB_REPOSITORY", "QuantStrategyLab/example") + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "123") monkeypatch.setattr(module, "staged_changes_present", lambda: True) - monkeypatch.setattr(module, "existing_pull_request", lambda branch: "") - def fake_run(command, *, capture=False): + def fake_run(command, *, capture=False, strip=True): commands.append(list(command)) - return "https://example.test/pr/2" if command[:3] == ["gh", "pr", "create"] else "" + if command[-3:] == ["--binary", "--full-index", "--cached"]: + assert strip is False + return "diff --git a/data/live/example.csv b/data/live/example.csv\n" + if command[-3:] == ["--cached", "--name-only", "--diff-filter=ACMR"]: + return "data/live/example.csv" + return "" monkeypatch.setattr(module, "run", fake_run) - result = module.publish("automation/generated", "Generated", "body", "commit") - - assert result == "Created generated-data PR: https://example.test/pr/2" - create = next(command for command in commands if command[:3] == ["gh", "pr", "create"]) - assert ["--head", "automation/generated"] == create[create.index("--head") : create.index("--head") + 2] - assert ["--base", "main"] == create[create.index("--base") : create.index("--base") + 2] + result = module.publish( + "automation/generated", + "Generated", + "body", + "commit", + handoff_dir=tmp_path, + ) + + assert result == "HUMAN_REQUIRED: generated publication handoff artifact." + assert commands == [ + ["git", "diff", "--binary", "--full-index", "--cached"], + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], + ] + + patch = tmp_path / "generated-live-output.patch" + receipt = json.loads((tmp_path / "publication-handoff.json").read_text(encoding="utf-8")) + assert patch.read_text(encoding="utf-8").startswith("diff --git") + assert patch.read_bytes().endswith(b"\n") + assert receipt == { + "schema_version": 1, + "status": "HUMAN_REQUIRED", + "reason_code": "PR_CREATION_IDENTITY_UNAVAILABLE", + "repository": "QuantStrategyLab/example", + "source_sha": "a" * 40, + "run_id": "123", + "target_branch": "main", + "proposed_branch": "automation/generated", + "title": "Generated", + "staged_paths": ["data/live/example.csv"], + "patch_path": "generated-live-output.patch", + "patch_sha256": module.hashlib.sha256(patch.read_bytes()).hexdigest(), + "next_action": "A maintainer must review and apply the patch through the protected branch process.", + } + + +def test_source_workflows_use_read_only_handoff_artifacts() -> None: + root = Path(__file__).parents[1] + workflows = { + "rss_source_pipeline.yml": "data/output/rss_source_pipeline", + "source_event_pipeline.yml": "data/output/source_event_pipeline", + } + + for filename, output_dir in workflows.items(): + text = (root / ".github" / "workflows" / filename).read_text(encoding="utf-8") + assert "permissions:\n contents: read" in text + assert "contents: write" not in text + assert "pull-requests: write" not in text + assert f"--handoff-dir {output_dir}" in text + assert "if: ${{ always() }}" in text