diff --git a/.github/workflows/sync-cloud-run-env.yml b/.github/workflows/sync-cloud-run-env.yml index 1a4c003..e3038d8 100644 --- a/.github/workflows/sync-cloud-run-env.yml +++ b/.github/workflows/sync-cloud-run-env.yml @@ -1415,8 +1415,10 @@ jobs: --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ --oidc-token-audience="${service_url}" \ --attempt-deadline=180s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ + --max-retry-attempts=3 \ + --min-backoff=120s \ + --max-backoff=300s \ + --max-retry-duration=900s \ --quiet else echo "Creating Cloud Scheduler probe ${probe_job_name} at ${desired_probe_schedule}." @@ -1430,8 +1432,10 @@ jobs: --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ --oidc-token-audience="${service_url}" \ --attempt-deadline=180s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ + --max-retry-attempts=3 \ + --min-backoff=120s \ + --max-backoff=300s \ + --max-retry-duration=900s \ --quiet fi @@ -1451,8 +1455,10 @@ jobs: --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ --oidc-token-audience="${service_url}" \ --attempt-deadline=180s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ + --max-retry-attempts=3 \ + --min-backoff=120s \ + --max-backoff=300s \ + --max-retry-duration=900s \ --quiet else echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}." @@ -1466,8 +1472,10 @@ jobs: --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \ --oidc-token-audience="${service_url}" \ --attempt-deadline=180s \ - --max-retry-attempts=0 \ - --max-retry-duration=0s \ + --max-retry-attempts=3 \ + --min-backoff=120s \ + --max-backoff=300s \ + --max-retry-duration=900s \ --quiet fi diff --git a/tests/test_scheduler_workflow.py b/tests/test_scheduler_workflow.py index 6e30eac..1b0931f 100644 --- a/tests/test_scheduler_workflow.py +++ b/tests/test_scheduler_workflow.py @@ -54,6 +54,26 @@ def test_main_scheduler_update_and_create_use_post_oidc() -> None: assert "--http-method=POST" in command_section assert '--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}"' in command_section assert '--oidc-token-audience="${service_url}"' in command_section + assert "--max-retry-attempts=3" not in command_section + + +def test_probe_and_precheck_retry_transient_capacity_errors() -> None: + workflow = Path(".github/workflows/sync-cloud-run-env.yml").read_text(encoding="utf-8") + for marker in ( + 'probe_job_name="${CLOUD_RUN_SERVICE}-probe-scheduler"', + 'precheck_job_name="${CLOUD_RUN_SERVICE}-precheck-scheduler"', + ): + section = workflow[workflow.index(marker) :] + # Stop before the next managed job block. + if "precheck_job_name=" in marker: + section = section[: section.index("managed_scheduler_jobs=")] + else: + section = section[: section.index("precheck_job_name=")] + assert "--max-retry-attempts=3" in section + assert "--min-backoff=120s" in section + assert "--max-backoff=300s" in section + assert "--max-retry-duration=900s" in section + assert "--max-retry-attempts=0" not in section def test_cloud_run_deploy_stays_private_and_serial() -> None: diff --git a/tests/test_sync_cloud_run_env_workflow.sh b/tests/test_sync_cloud_run_env_workflow.sh index 6a5367e..bc1efc5 100644 --- a/tests/test_sync_cloud_run_env_workflow.sh +++ b/tests/test_sync_cloud_run_env_workflow.sh @@ -241,6 +241,12 @@ grep -Fq 'probe_job_name="${CLOUD_RUN_SERVICE}-probe-scheduler"' "$workflow_file grep -Fq 'probe_uri="${service_url}/probe"' "$workflow_file" grep -Fq 'precheck_job_name="${CLOUD_RUN_SERVICE}-precheck-scheduler"' "$workflow_file" grep -Fq 'precheck_uri="${service_url}/dry-run"' "$workflow_file" +# probe/precheck may collide on maxScale=1; retry transient 429 capacity aborts. +# Keep /run without retries to avoid duplicate live submits. +grep -Fq -- '--max-retry-attempts=3' "$workflow_file" +grep -Fq -- '--min-backoff=120s' "$workflow_file" +grep -Fq -- '--max-backoff=300s' "$workflow_file" +grep -Fq -- '--max-retry-duration=900s' "$workflow_file" grep -Fq 'managed_scheduler_jobs=("${job_name}" "${probe_job_name}" "${precheck_job_name}")' "$workflow_file" grep -Fq 'gcloud scheduler jobs resume "${managed_job_name}"' "$workflow_file" grep -Fq 'gcloud scheduler jobs pause "${managed_job_name}"' "$workflow_file"