Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions .github/workflows/execution-report-heartbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,14 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6

- name: Setup uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0

- name: Install locked runtime dependencies
run: |
set -euo pipefail
uv sync --frozen --no-dev

- name: Authenticate to Google Cloud
id: gcp_auth_primary
continue-on-error: true
Expand All @@ -89,14 +97,8 @@ jobs:
- name: Set up gcloud
uses: google-github-actions/setup-gcloud@v3

- name: Install market calendar
continue-on-error: true
run: >-
python -m pip install --disable-pip-version-check
--retries 3 --timeout 30 "pandas-market-calendars==5.4.0"

- name: Check recent execution report
run: python scripts/execution_report_heartbeat.py
run: uv run --no-sync python scripts/execution_report_heartbeat.py

- name: Publish read-only runtime execution evidence
uses: QuantStrategyLab/QuantRuntimeSettings/actions/publish-runtime-execution-evidence@1eb72e4be2c20f991be5e10153ae046ba0fde496
Expand Down
15 changes: 14 additions & 1 deletion .github/workflows/runtime-guard.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,19 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6

- name: Setup Python
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1
with:
python-version: "3.12"

- name: Setup uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0

- name: Install locked runtime dependencies
run: |
set -euo pipefail
uv sync --frozen --no-dev

- name: Authenticate to Google Cloud
id: gcp_auth_primary
continue-on-error: true
Expand All @@ -86,4 +99,4 @@ jobs:
uses: google-github-actions/setup-gcloud@v3

- name: Check Cloud Scheduler and Cloud Run logs
run: python scripts/cloud_run_runtime_guard.py
run: uv run --no-sync python scripts/cloud_run_runtime_guard.py
22 changes: 12 additions & 10 deletions .github/workflows/runtime-target-lifecycle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,14 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v6

- name: Setup uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0

- name: Install locked runtime dependencies
run: |
set -euo pipefail
uv sync --frozen --no-dev

- name: Resolve declared no-order target state
id: target_state
run: |
Expand Down Expand Up @@ -96,26 +104,20 @@ jobs:
run: |
set -uo pipefail
set +e
python scripts/cloud_run_runtime_guard.py > runtime-guard.log 2>&1
uv run --no-sync python scripts/cloud_run_runtime_guard.py > runtime-guard.log 2>&1
exit_code=$?
set -e
cat runtime-guard.log
if [ "$exit_code" -eq 0 ]; then
status=pass
elif grep -Eqi 'log query failed|internal error|network|timed out|unable to list' runtime-guard.log; then
elif grep -Eqi 'log query failed|import|traceback|internal error|network|timed out|unable to list' runtime-guard.log; then
status=unavailable
else
status=attention
fi
echo "status=$status" >> "$GITHUB_OUTPUT"
exit 0

- name: Install market calendar for enabled heartbeat
if: ${{ steps.target_state.outputs.configured_state == 'enabled' }}
run: >-
python -m pip install --disable-pip-version-check
--retries 3 --timeout 30 "pandas-market-calendars==5.4.0"

- name: Check enabled execution heartbeat without notification side effects
id: execution_heartbeat
env:
Expand All @@ -132,15 +134,15 @@ jobs:
fi

set +e
python scripts/execution_report_heartbeat.py > execution-heartbeat.log 2>&1
uv run --no-sync python scripts/execution_report_heartbeat.py > execution-heartbeat.log 2>&1
exit_code=$?
set -e
cat execution-heartbeat.log
if [ "$exit_code" -eq 0 ] && grep -qi 'heartbeat skipped' execution-heartbeat.log; then
status=not_due
elif [ "$exit_code" -eq 0 ]; then
status=pass
elif grep -Eqi 'gcloud|storage|network|timed out|internal error|unable to list' execution-heartbeat.log; then
elif grep -Eqi 'gcloud|storage|import|traceback|network|timed out|internal error|unable to list' execution-heartbeat.log; then
status=unavailable
else
status=attention
Expand Down
71 changes: 70 additions & 1 deletion tests/test_runtime_monitor_workflows.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import re
from pathlib import Path


Expand All @@ -14,7 +15,38 @@ def test_execution_report_heartbeat_has_market_neutral_daily_schedule() -> None:
assert "RUNTIME_HEARTBEAT_SCHEDULER_LOCATION:" in workflow
assert "CLOUD_SCHEDULER_MAIN_TIME:" in workflow
assert "EXECUTION_REPORT_GCS_URI:" in workflow
assert "pandas-market-calendars==5.4.0" in workflow
assert "pandas-market-calendars==5.4.0" not in workflow


def test_qpk_dependent_heartbeats_use_one_locked_uv_runtime_per_job() -> None:
setup_uv = "astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9"
qpk_scripts = {
"execution-report-heartbeat.yml": ("scripts/execution_report_heartbeat.py",),
"runtime-target-lifecycle.yml": (
"scripts/cloud_run_runtime_guard.py",
"scripts/execution_report_heartbeat.py",
),
}

for name, scripts in qpk_scripts.items():
workflow = (ROOT / ".github/workflows" / name).read_text()

assert workflow.count(setup_uv) == 1
assert workflow.count("uv sync --frozen --no-dev") == 1
assert "pandas-market-calendars==5.4.0" not in workflow
assert "python -m pip install" not in workflow
assert "actions/setup-python@" not in workflow
for script in scripts:
script_lines = [line for line in workflow.splitlines() if script in line]
assert script_lines
assert all(f"uv run --no-sync python {script}" in line for line in script_lines)


def test_lifecycle_import_failures_are_unavailable() -> None:
workflow = (ROOT / ".github/workflows/runtime-target-lifecycle.yml").read_text()

assert workflow.count("status=unavailable") == 2
assert workflow.count("|import|traceback|") == 2


def test_runtime_monitor_workflows_retry_gcp_authentication() -> None:
Expand All @@ -25,3 +57,40 @@ def test_runtime_monitor_workflows_retry_gcp_authentication() -> None:
assert "id: gcp_auth_primary" in workflow
assert "continue-on-error: true" in workflow
assert "steps.gcp_auth_primary.outcome == 'failure'" in workflow


def test_runtime_guard_callers_use_locked_uv_runtime_before_authentication() -> None:
workflow_root = ROOT / ".github/workflows"
callers = sorted(
path
for pattern in ("*.yml", "*.yaml")
for path in workflow_root.rglob(pattern)
if "scripts/cloud_run_runtime_guard.py" in path.read_text()
)

assert callers
setup_uv = "astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9"
setup_python = "actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1"
guard_command = "uv run --no-sync python scripts/cloud_run_runtime_guard.py"
setup_python_lines = []
for path in callers:
workflow = path.read_text()

assert setup_uv in workflow
assert "uv sync --frozen --no-dev" in workflow
assert guard_command in workflow
assert not re.search(r"pip install[^\n]*\buv\b", workflow)
assert "actions/setup-python@v6" not in workflow
setup_python_lines.extend(
line for line in workflow.splitlines() if "actions/setup-python@" in line
)
assert all(
guard_command in line
for line in workflow.splitlines()
if "scripts/cloud_run_runtime_guard.py" in line
)
assert workflow.index(setup_uv) < workflow.index("google-github-actions/auth@v3")
assert workflow.index("uv sync --frozen --no-dev") < workflow.index(
"google-github-actions/auth@v3"
)
assert setup_python_lines == [f" uses: {setup_python}"]