fix(scheduler): retry precheck/probe on capacity 429

.github/workflows/sync-cloud-run-env.yml

@@ -1415,8 +1415,10 @@ jobs:
               --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
               --oidc-token-audience="${service_url}" \
               --attempt-deadline=180s \
-              --max-retry-attempts=0 \
-              --max-retry-duration=0s \
+              --max-retry-attempts=3 \
+              --min-backoff=120s \
+              --max-backoff=300s \
+              --max-retry-duration=900s \
               --quiet
           else
             echo "Creating Cloud Scheduler probe ${probe_job_name} at ${desired_probe_schedule}."
@@ -1430,8 +1432,10 @@ jobs:
               --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
               --oidc-token-audience="${service_url}" \
               --attempt-deadline=180s \
-              --max-retry-attempts=0 \
-              --max-retry-duration=0s \
+              --max-retry-attempts=3 \
+              --min-backoff=120s \
+              --max-backoff=300s \
+              --max-retry-duration=900s \
               --quiet
           fi
 
@@ -1451,8 +1455,10 @@ jobs:
               --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
               --oidc-token-audience="${service_url}" \
               --attempt-deadline=180s \
-              --max-retry-attempts=0 \
-              --max-retry-duration=0s \
+              --max-retry-attempts=3 \
+              --min-backoff=120s \
+              --max-backoff=300s \
+              --max-retry-duration=900s \
               --quiet
           else
             echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}."
@@ -1466,8 +1472,10 @@ jobs:
               --oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}" \
               --oidc-token-audience="${service_url}" \
               --attempt-deadline=180s \
-              --max-retry-attempts=0 \
-              --max-retry-duration=0s \
+              --max-retry-attempts=3 \
+              --min-backoff=120s \
+              --max-backoff=300s \
+              --max-retry-duration=900s \
               --quiet
           fi
 

tests/test_scheduler_workflow.py

@@ -54,6 +54,26 @@ def test_main_scheduler_update_and_create_use_post_oidc() -> None:
         assert "--http-method=POST" in command_section
         assert '--oidc-service-account-email="${GCP_SCHEDULER_SERVICE_ACCOUNT}"' in command_section
         assert '--oidc-token-audience="${service_url}"' in command_section
+        assert "--max-retry-attempts=3" not in command_section
+
+
+def test_probe_and_precheck_retry_transient_capacity_errors() -> None:
+    workflow = Path(".github/workflows/sync-cloud-run-env.yml").read_text(encoding="utf-8")
+    for marker in (
+        'probe_job_name="${CLOUD_RUN_SERVICE}-probe-scheduler"',
+        'precheck_job_name="${CLOUD_RUN_SERVICE}-precheck-scheduler"',
+    ):
+        section = workflow[workflow.index(marker) :]
+        # Stop before the next managed job block.
+        if "precheck_job_name=" in marker:
+            section = section[: section.index("managed_scheduler_jobs=")]
+        else:
+            section = section[: section.index("precheck_job_name=")]
+        assert "--max-retry-attempts=3" in section
+        assert "--min-backoff=120s" in section
+        assert "--max-backoff=300s" in section
+        assert "--max-retry-duration=900s" in section
+        assert "--max-retry-attempts=0" not in section
 
 
 def test_cloud_run_deploy_stays_private_and_serial() -> None:

tests/test_sync_cloud_run_env_workflow.sh

@@ -241,6 +241,12 @@ grep -Fq 'probe_job_name="${CLOUD_RUN_SERVICE}-probe-scheduler"' "$workflow_file
 grep -Fq 'probe_uri="${service_url}/probe"' "$workflow_file"
 grep -Fq 'precheck_job_name="${CLOUD_RUN_SERVICE}-precheck-scheduler"' "$workflow_file"
 grep -Fq 'precheck_uri="${service_url}/dry-run"' "$workflow_file"
+# probe/precheck may collide on maxScale=1; retry transient 429 capacity aborts.
+# Keep /run without retries to avoid duplicate live submits.
+grep -Fq -- '--max-retry-attempts=3' "$workflow_file"
+grep -Fq -- '--min-backoff=120s' "$workflow_file"
+grep -Fq -- '--max-backoff=300s' "$workflow_file"
+grep -Fq -- '--max-retry-duration=900s' "$workflow_file"
 grep -Fq 'managed_scheduler_jobs=("${job_name}" "${probe_job_name}" "${precheck_job_name}")' "$workflow_file"
 grep -Fq 'gcloud scheduler jobs resume "${managed_job_name}"' "$workflow_file"
 grep -Fq 'gcloud scheduler jobs pause "${managed_job_name}"' "$workflow_file"