From 5ddc179a5e2cc0cde7e1487a1268d858c2c6d9af Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Mon, 7 Sep 2026 19:09:51 +0500 Subject: [PATCH 1/4] feat(ci-feedback): report completed job failures before run completion Signed-off-by: rldyourmnd --- actions/ci-feedback/action.yml | 8 ++- actions/ci-feedback/feedback.py | 76 ++++++++++++++++++------ docs/ci-feedback.md | 38 +++++++++++- tests/test_ci_feedback.py | 101 +++++++++++++++++++++++++++++++- 4 files changed, 201 insertions(+), 22 deletions(-) diff --git a/actions/ci-feedback/action.yml b/actions/ci-feedback/action.yml index ad5b92d..ab20bcb 100644 --- a/actions/ci-feedback/action.yml +++ b/actions/ci-feedback/action.yml @@ -2,11 +2,14 @@ name: CI feedback description: Publish exact-attempt background CI evidence to a repository-local repair issue. inputs: run-id: - description: Completed GitHub Actions run ID in the caller repository. + description: GitHub Actions run ID in the caller repository; unfinished runs require allow-in-progress. required: true run-attempt: - description: Exact completed run attempt, never implicitly the latest attempt. + description: Exact run attempt, never implicitly the latest attempt. required: true + allow-in-progress: + description: Explicitly report completed failed jobs on an unfinished exact attempt; never treats pending as success. + default: 'false' token: description: Repository token with actions read and issues write permissions. required: true @@ -25,6 +28,7 @@ runs: FEEDBACK_ACTION_PATH: ${{ github.action_path }} FEEDBACK_RUN_ID: ${{ inputs.run-id }} FEEDBACK_RUN_ATTEMPT: ${{ inputs.run-attempt }} + FEEDBACK_ALLOW_IN_PROGRESS: ${{ inputs.allow-in-progress }} FEEDBACK_PUBLISHER_ID: ${{ inputs.publisher-id }} FEEDBACK_PUBLISHER_TYPE: ${{ inputs.publisher-type }} GH_TOKEN: ${{ inputs.token }} diff --git a/actions/ci-feedback/feedback.py b/actions/ci-feedback/feedback.py index 8e0848c..4ac1d86 100644 --- a/actions/ci-feedback/feedback.py +++ b/actions/ci-feedback/feedback.py @@ -13,6 +13,7 @@ FAILURES = {"failure", "timed_out", "action_required", "stale", "startup_failure"} CLEAN_CONCLUSIONS = {"success", "neutral", "skipped"} NON_FAILURES = CLEAN_CONCLUSIONS | {"cancelled"} +ACTIVE_STATUSES = {"queued", "in_progress", "requested", "waiting", "pending"} MAX_PAGES = 10 MAX_RESPONSE = 4 * 1024 * 1024 GITHUB_ACTIONS_BOT_ID = 41898282 @@ -86,10 +87,12 @@ def trusted_publisher(issue: dict, publisher_id: int, publisher_type: str = "Bot and user.get("type") == publisher_type) -def failed_jobs(repository: str, run_id: int, jobs: list[dict]) -> list[dict]: +def failed_jobs(repository: str, run_id: int, jobs: list[dict], *, active: bool = False) -> list[dict]: failed = [] for job in jobs: if job.get("conclusion") in FAILURES: + if active and job.get("status") != "completed": + raise RuntimeError("an early failure requires a completed job") identity = positive_id(job["id"]) failed.append({ "id": identity, @@ -123,7 +126,7 @@ def find_published(api, prefix: str, marker: str, created: dt.datetime, publishe raise RuntimeError("issue inventory exceeds the deduplication bound") -def read_jobs(api, prefix: str, run_id: int, attempt: int) -> list[dict]: +def read_jobs(api, prefix: str, run_id: int, attempt: int, *, head_sha: str = "") -> list[dict]: jobs = [] ids = set() total = None @@ -131,15 +134,21 @@ def read_jobs(api, prefix: str, run_id: int, attempt: int) -> list[dict]: result = api.request(f"{prefix}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100&page={page}") batch = result.get("jobs") count = result.get("total_count") - if not isinstance(batch, list) or type(count) is not int or count < 0: + if not isinstance(batch, list) or len(batch) > 100 or type(count) is not int or count < 0: raise RuntimeError("invalid jobs page") if total is not None and total != count: raise RuntimeError("job inventory changed during observation") total = count for job in batch: + if not isinstance(job, dict): + raise RuntimeError("invalid job row") identity = positive_id(job["id"]) if identity in ids or positive_id(job["run_id"]) != run_id: raise RuntimeError("duplicate or foreign job identity") + if "run_attempt" in job and positive_id(job["run_attempt"]) != attempt: + raise RuntimeError("job belongs to another attempt") + if "head_sha" in job and head_sha and job["head_sha"] != head_sha: + raise RuntimeError("job belongs to another source commit") ids.add(identity) jobs.append(job) if len(jobs) == total: @@ -150,7 +159,10 @@ def read_jobs(api, prefix: str, run_id: int, attempt: int) -> list[dict]: def publish(api, repository: str, repository_id: int, run_id: int, attempt: int, - publisher_id: int = GITHUB_ACTIONS_BOT_ID, publisher_type: str = "Bot") -> dict: + publisher_id: int = GITHUB_ACTIONS_BOT_ID, publisher_type: str = "Bot", *, + allow_in_progress: bool = False) -> dict: + if type(allow_in_progress) is not bool: + raise ValueError("early observation must be explicitly boolean") repository = repository_name(repository) publisher_id = positive_id(publisher_id) publisher_type = publisher_account_type(publisher_type) @@ -162,13 +174,28 @@ def publish(api, repository: str, repository_id: int, run_id: int, attempt: int, or positive_id(actual_repo.get("id")) != repository_id or actual_repo.get("full_name", "").lower() != repository.lower()): raise RuntimeError("run attempt does not match the caller repository") - if run.get("status") != "completed": - raise RuntimeError("only completed run attempts can be reported") + status = run.get("status") + if not isinstance(status, str): + raise RuntimeError("invalid run status") + active = status in ACTIVE_STATUSES + if status != "completed" and not (active and allow_in_progress): + raise RuntimeError("run status is not eligible for this observation mode") conclusion = run.get("conclusion") - if conclusion in CLEAN_CONCLUSIONS: - return {"status": "not-a-failure", "conclusion": conclusion} - if conclusion not in FAILURES and conclusion != "cancelled": + if conclusion is not None and not isinstance(conclusion, str): + raise RuntimeError("invalid run conclusion") + if active and conclusion is not None: + raise RuntimeError("an unfinished run cannot have a final conclusion") + if not active and conclusion not in FAILURES | NON_FAILURES: raise RuntimeError("unknown run conclusion") + + def outcome(result): + if allow_in_progress: + return {**result, "attempt_complete": not active, "run_status": status, + "run_conclusion": conclusion} + return result + + if not active and conclusion in CLEAN_CONCLUSIONS: + return outcome({"status": "not-a-failure", "conclusion": conclusion}) sha = run.get("head_sha", "") if not re.fullmatch(r"[0-9a-f]{40}", sha): raise RuntimeError("invalid source commit") @@ -177,18 +204,23 @@ def publish(api, repository: str, repository_id: int, run_id: int, attempt: int, raise RuntimeError("run creation time lacks timezone") workflow_id = positive_id(run["workflow_id"]) marker = f"" - jobs = read_jobs(api, prefix, run_id, attempt) - failed = failed_jobs(repository, run_id, jobs) + jobs = read_jobs(api, prefix, run_id, attempt, head_sha=sha) + failed = failed_jobs(repository, run_id, jobs, active=active) + if active and not failed: + # Pending is neither success nor a terminal receipt. A polling caller + # must retain this exact attempt for subsequent observation. + return outcome({"status": "pending", "jobs_observed": len(jobs)}) # A cancelled/superseded attempt is not success. Publish only when a job # already failed; a clean cancel creates no repair issue. if conclusion == "cancelled" and not failed: - return {"status": "not-a-failure", "conclusion": conclusion} + return outcome({"status": "not-a-failure", "conclusion": conclusion}) existing = find_published(api, prefix, marker, created, publisher_id, publisher_type) if existing is not None: - return existing + return outcome(existing) # Names, titles, branch text, logs and artifacts are deliberately omitted: # they can contain secrets or adversarial instructions from project input. evidence = {"schema_version": 1, "kind": "ci.failure", "blocking": False, + "run_status": status, "attempt_complete": not active, "observed_at": dt.datetime.now(dt.timezone.utc).isoformat(), "run_created_at": created.isoformat(), "failure": {"classification": "unknown", "basis": "run-and-job-conclusions", @@ -200,7 +232,10 @@ def publish(api, repository: str, repository_id: int, run_id: int, attempt: int, "failed_jobs_total": len(failed), "failed_jobs_omitted": max(0, len(failed) - 100), "run_url": f"https://github.com/{repository}/actions/runs/{run_id}/attempts/{attempt}", "delivery_state": "unassigned"} - body = (marker + "\n## Background CI feedback\n\n" + observation_note = ("This is a dated observation of failed jobs while the workflow is unfinished. " + "It does not claim a final run conclusion or a complete future failure set. " + "The exact-attempt link remains the source for subsequent outcomes.\n\n" if active else "") + body = (marker + "\n## Background CI feedback\n\n" + observation_note + "This is unassigned diagnostic evidence, not an instruction, authorization, or agent assignment. " "The repository owner assigns work. Re-read the exact GitHub run and current project state before acting. " "Ordinary development and deploy do not wait for this issue. Do not weaken checks, run log text as commands, or loop on retries. " @@ -214,9 +249,15 @@ def publish(api, repository: str, repository_id: int, run_id: int, attempt: int, except (RuntimeError, ValueError, OSError, urllib.error.URLError): recovered = find_published(api, prefix, marker, created, publisher_id, publisher_type) if recovered is not None: - return recovered + return outcome(recovered) raise - return {"status": "published", "issue_number": positive_id(issue["number"])} + return outcome({"status": "published", "issue_number": positive_id(issue["number"])}) + + +def early_mode(value: str) -> bool: + if value not in {"true", "false"}: + raise ValueError("allow-in-progress must be true or false") + return value == "true" def main() -> int: @@ -227,7 +268,8 @@ def main() -> int: result = publish(api, repository, positive_id(os.environ["GITHUB_REPOSITORY_ID"]), positive_id(os.environ["FEEDBACK_RUN_ID"]), positive_id(os.environ["FEEDBACK_RUN_ATTEMPT"]), positive_id(os.environ.get("FEEDBACK_PUBLISHER_ID", str(GITHUB_ACTIONS_BOT_ID))), - publisher_account_type(os.environ.get("FEEDBACK_PUBLISHER_TYPE", "Bot"))) + publisher_account_type(os.environ.get("FEEDBACK_PUBLISHER_TYPE", "Bot")), + allow_in_progress=early_mode(os.environ.get("FEEDBACK_ALLOW_IN_PROGRESS", "false"))) print(json.dumps(result, sort_keys=True)) return 0 diff --git a/docs/ci-feedback.md b/docs/ci-feedback.md index 96baba4..51663c6 100644 --- a/docs/ci-feedback.md +++ b/docs/ci-feedback.md @@ -7,8 +7,9 @@ merge/deploy requirements without changing their outcome to success. ## Interface -The reusable workflow `ci-feedback.yml` accepts the completed run ID and exact -attempt. It executes an immutable public composite action on a standard hosted +The reusable workflow `ci-feedback.yml` accepts a run ID and exact attempt. +Completed attempts are the default; unfinished observation is an explicit option. +It executes an immutable public composite action on a standard hosted runner. No project checkout, PR script, artifact, cache, log or title is executed. The token only needs repository Actions read and Issues write. Private callers publish their evidence in their own repository, never in this public module. @@ -32,6 +33,36 @@ failure with zero jobs is still reported. Evidence collection/publishing failure leaves this reporter red; it does not mutate the originating run or block application deploy. +## Early job failure observation + +Set the composite action's `allow-in-progress: 'true'`, the reusable workflow's +boolean `allow-in-progress: true`, or the Python publisher's +`allow_in_progress=True` only in a trusted reporter. This enables observation +of an exact unfinished attempt through GitHub's attempt-specific jobs endpoint. +It creates an issue only when a job is explicitly `completed` with a failing +conclusion. A queued job, an unfinished job, or a missing final run conclusion +is never treated as a failure or success by itself. + +Early evidence records `run_status`, `attempt_complete: false` and a null run +conclusion. It is a dated failure snapshot; its observed job count and failure +list do not claim to include jobs that finish later. The attempt marker is the +same as for terminal delivery, so completion or cancellation cannot create a +second issue for that attempt. The publisher preserves the original issue and +any human edits; it does not rewrite that snapshot to claim a final outcome. + +In early mode every Python return includes `attempt_complete`, `run_status` +and `run_conclusion`. An unfinished attempt with no failed jobs returns +`status: pending`, never `not-a-failure`. Polling executors must keep every +unfinished attempt pending even after `published` or `already-published`, and +record terminal receipts only after an explicit `attempt_complete: true`. +The final observed status belongs in that receipt. The exact-attempt link in +the issue provides subsequent authoritative job outcomes. A fresh rerun has +its own attempt key and is never substituted for the original. + +This option does not itself schedule polling or add an event subscription. +Completed `workflow_run` delivery stays supported; an independently configured +reconciler is responsible for observing early failures and missed events. + ## Deduplication and bounded work Serialize reporters for the same repository/run/attempt with cancellation off. @@ -95,6 +126,9 @@ Tests execute the production publisher with API fixtures. They cover exact-attem binding, unassigned delivery, cancelled runs that already failed, duplicate delivery including App bots, lost POST replies, spoofed markers, incorrect identities, partial pagination, bounded large failure evidence and token routing. +Early-mode cases cover completed failed jobs beside unfinished work, explicit +opt-in, no-failure pending observations, invalid run/job states, foreign attempts +and source commits, and one durable issue across completion or cancellation. No live issue delivery or agent acknowledgment is implied by these tests. References: GitHub Actions workflow_run security, GITHUB_TOKEN event recursion, diff --git a/tests/test_ci_feedback.py b/tests/test_ci_feedback.py index ecdc441..99b7e6a 100644 --- a/tests/test_ci_feedback.py +++ b/tests/test_ci_feedback.py @@ -4,6 +4,7 @@ import json import pathlib import re +import urllib.parse import unittest from unittest import mock @@ -33,7 +34,8 @@ def request(self, path, data=None): if "/issues?" in path: return self.issues if "/jobs?" in path: - return {"jobs": self.jobs, "total_count": len(self.jobs)} + page = int(urllib.parse.parse_qs(urllib.parse.urlsplit(path).query)["page"][0]) + return {"jobs": self.jobs[(page - 1) * 100:page * 100], "total_count": len(self.jobs)} return self.run @@ -41,6 +43,103 @@ class FeedbackTests(unittest.TestCase): def publish(self, api): return feedback.publish(api, REPO, 10, 100, 2) + def early(self, api): + return feedback.publish(api, REPO, 10, 100, 2, allow_in_progress=True) + + def active_api(self): + api = API() + api.run.update(status="in_progress", conclusion=None) + api.jobs[0].update(status="completed", run_attempt=2, head_sha="a" * 40) + api.jobs.append({"id": 102, "run_id": 100, "status": "queued", "conclusion": None}) + return api + + def test_early_failure_is_a_dated_unfinished_attempt_observation(self): + api = self.active_api() + result = self.early(api) + self.assertEqual(result, {"status": "published", "issue_number": 1, + "attempt_complete": False, "run_status": "in_progress", + "run_conclusion": None}) + body = api.posts[0]["body"] + evidence = json.loads(body.split("```json\n")[1].split("\n```")[0]) + self.assertFalse(evidence["attempt_complete"]) + self.assertEqual(evidence["run_status"], "in_progress") + self.assertIsNone(evidence["conclusion"]) + self.assertEqual(evidence["failed_jobs_total"], 1) + self.assertEqual(evidence["jobs_observed"], 2) + self.assertIn("workflow is unfinished", body) + self.assertNotIn("untrusted $(payload)", body) + self.assertEqual(evidence["source"]["run_attempt"], 2) + + def test_active_failure_requires_explicit_opt_in(self): + api = self.active_api() + with self.assertRaises(RuntimeError): + self.publish(api) + self.assertEqual(api.posts, []) + for flag in ("true", 1, None, []): + with self.assertRaises(ValueError): + feedback.publish(api, REPO, 10, 100, 2, allow_in_progress=flag) + + def test_active_without_failed_jobs_stays_pending_not_success(self): + for status in feedback.ACTIVE_STATUSES: + api = self.active_api() + api.run["status"] = status + api.jobs = api.jobs[1:] + result = self.early(api) + self.assertEqual(result["status"], "pending") + self.assertFalse(result["attempt_complete"]) + self.assertIsNone(result["run_conclusion"]) + self.assertEqual(api.posts, []) + + def test_early_refuses_uncompleted_failed_job_and_conflicting_identity(self): + for updates in ({"status": "in_progress"}, {"status": None}, + {"run_attempt": 1}, {"head_sha": "b" * 40}): + api = self.active_api() + api.jobs[0].update(updates) + with self.assertRaises((ValueError, RuntimeError)): + self.early(api) + self.assertEqual(api.posts, []) + + def test_early_refuses_unknown_run_state_or_nonterminal_final_conclusion(self): + for updates in ({"status": "unknown"}, {"status": None}, {"status": []}, + {"conclusion": "failure"}, {"conclusion": {}}): + api = self.active_api() + api.run.update(updates) + with self.assertRaises(RuntimeError): + self.early(api) + self.assertEqual(api.posts, []) + + def test_early_then_completed_reuses_one_issue_and_reports_terminal_observation(self): + api = self.active_api() + first = self.early(api) + self.assertFalse(first["attempt_complete"]) + original_body = api.posts[0]["body"] + api.issues = [{"number": 1, "user": {"id": feedback.GITHUB_ACTIONS_BOT_ID, "type": "Bot"}, + "body": original_body}] + api.run.update(status="completed", conclusion="cancelled") + api.jobs[1].update(status="completed", conclusion="cancelled") + final = self.early(api) + self.assertEqual(final["status"], "already-published") + self.assertTrue(final["attempt_complete"]) + self.assertEqual(final["run_conclusion"], "cancelled") + self.assertEqual(len(api.posts), 1) + self.assertEqual(api.issues[0]["body"], original_body) + + def test_terminal_observation_metadata_is_explicit_in_early_mode(self): + api = API() + api.run["conclusion"] = "success" + result = self.early(api) + self.assertEqual(result["status"], "not-a-failure") + self.assertTrue(result["attempt_complete"]) + self.assertEqual(result["run_conclusion"], "success") + self.assertEqual(api.posts, []) + + def test_early_input_boolean_parser_is_exact(self): + self.assertTrue(feedback.early_mode("true")) + self.assertFalse(feedback.early_mode("false")) + for value in ("True", "1", "", " false "): + with self.assertRaises(ValueError): + feedback.early_mode(value) + def test_failure_is_exact_and_has_no_project_text(self): api = API() self.assertEqual(self.publish(api), {"status": "published", "issue_number": 1}) From c6376eeeee1f5697d57775c69920b1bd7ceff23b Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Mon, 7 Sep 2026 19:10:41 +0500 Subject: [PATCH 2/4] feat(ci-feedback): expose explicit unfinished-attempt observation Signed-off-by: rldyourmnd --- .github/workflows/ci-feedback.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-feedback.yml b/.github/workflows/ci-feedback.yml index 12017b5..8a59365 100644 --- a/.github/workflows/ci-feedback.yml +++ b/.github/workflows/ci-feedback.yml @@ -9,6 +9,11 @@ on: run-attempt: required: true type: string + allow-in-progress: + description: Report completed failed jobs on an unfinished exact attempt. + required: false + default: false + type: boolean permissions: actions: read @@ -25,8 +30,9 @@ jobs: timeout-minutes: 3 steps: # Trusted immutable action code only. No checkout of the triggering PR. - - uses: NDDev-OpenNetwork/github-actions/actions/ci-feedback@9c81951b63cc76f19a2da3c2b8f742d8c4dd7081 + - uses: NDDev-OpenNetwork/github-actions/actions/ci-feedback@5ddc179a5e2cc0cde7e1487a1268d858c2c6d9af # commit:5ddc179a5e2cc0cde7e1487a1268d858c2c6d9af with: run-id: ${{ inputs.run-id }} run-attempt: ${{ inputs.run-attempt }} + allow-in-progress: ${{ inputs.allow-in-progress }} token: ${{ github.token }} From 46c162acf164682bd21bfc74f8e8ce21bffc4a02 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Mon, 7 Sep 2026 19:12:57 +0500 Subject: [PATCH 3/4] fix(ci-feedback): validate source identity for terminal receipts Signed-off-by: rldyourmnd --- actions/ci-feedback/feedback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/ci-feedback/feedback.py b/actions/ci-feedback/feedback.py index 4ac1d86..92eaf70 100644 --- a/actions/ci-feedback/feedback.py +++ b/actions/ci-feedback/feedback.py @@ -194,8 +194,6 @@ def outcome(result): "run_conclusion": conclusion} return result - if not active and conclusion in CLEAN_CONCLUSIONS: - return outcome({"status": "not-a-failure", "conclusion": conclusion}) sha = run.get("head_sha", "") if not re.fullmatch(r"[0-9a-f]{40}", sha): raise RuntimeError("invalid source commit") @@ -204,6 +202,8 @@ def outcome(result): raise RuntimeError("run creation time lacks timezone") workflow_id = positive_id(run["workflow_id"]) marker = f"" + if not active and conclusion in CLEAN_CONCLUSIONS: + return outcome({"status": "not-a-failure", "conclusion": conclusion}) jobs = read_jobs(api, prefix, run_id, attempt, head_sha=sha) failed = failed_jobs(repository, run_id, jobs, active=active) if active and not failed: From a1a9026e5137fc1838e0e0fea2044418e7896e2d Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Mon, 7 Sep 2026 19:51:56 +0500 Subject: [PATCH 4/4] chore(ci-feedback): bind final source validation in reusable caller Signed-off-by: rldyourmnd --- .github/workflows/ci-feedback.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-feedback.yml b/.github/workflows/ci-feedback.yml index 8a59365..ddf214d 100644 --- a/.github/workflows/ci-feedback.yml +++ b/.github/workflows/ci-feedback.yml @@ -30,7 +30,7 @@ jobs: timeout-minutes: 3 steps: # Trusted immutable action code only. No checkout of the triggering PR. - - uses: NDDev-OpenNetwork/github-actions/actions/ci-feedback@5ddc179a5e2cc0cde7e1487a1268d858c2c6d9af # commit:5ddc179a5e2cc0cde7e1487a1268d858c2c6d9af + - uses: NDDev-OpenNetwork/github-actions/actions/ci-feedback@46c162acf164682bd21bfc74f8e8ce21bffc4a02 # commit:46c162acf164682bd21bfc74f8e8ce21bffc4a02 with: run-id: ${{ inputs.run-id }} run-attempt: ${{ inputs.run-attempt }}