fix(garm): retire excess idle online JIT registrations

.gds/bundle.lock.yaml

@@ -10,7 +10,7 @@ bundle:
   attestation_identity_digest: "sha256:e1b787c63de916f750e2cbcbf664967669024e1062964515f612b2c826c9e10d"
 
 projection:
-  input_digest: "sha256:460039c6ea070da7eb3a88a53f5c5cfcc7bf852209b4f7949411b6af43c442bc"
+  input_digest: "sha256:16d00b8c8cfe471cc00e14ab983f28cbed63448fc13f2f7f35977d827a172f11"
   output_digest: "sha256:6df8425d9b3ebe48a39fad7d21493444364c697256721cd692ea35e1658abf0e"
   files:
     - path: ".gds/compiled-policy.json"

.gds/repository.yaml

@@ -77,8 +77,7 @@ verification:
     - "lint"
     - "test"
     - "build"
-  required_contexts:
-    - "Gate"
+  required_contexts: []
 
 agent:
   context_profile: "project-default"

.github/branch-protection.yaml

@@ -1,7 +1,6 @@
 branch: main
 required_status_checks:
-  strict: true
-  contexts:
-    - Gate
+  strict: false
+  contexts: []
 required_pull_request_reviews: {}
 enforce_admins: true

.github/rulesets/branch-main.json

@@ -25,14 +25,6 @@
         "dismissal_restriction": {"enabled": false, "allowed_actors": []},
         "allowed_merge_methods": ["merge"]
       }
-    },
-    {
-      "type": "required_status_checks",
-      "parameters": {
-        "strict_required_status_checks_policy": true,
-        "do_not_enforce_on_create": false,
-        "required_status_checks": [{"context": "Gate"}]
-      }
     }
   ]
 }

AGENTS.md

@@ -14,3 +14,7 @@ identity, state/schema integrity, one-job isolation and release provenance.
 Formatting and generated artifacts should be repaired automatically.
 
 Run `make verify` after changes. Preserve unrelated worktree changes.
+
+Ordinary merge does not wait on a required general CI status check. The `Gate`
+job still runs as truthful advisory evidence. Live GitHub settings are not
+applied from this tree.

CHANGELOG.md

@@ -2,6 +2,23 @@
 
 ## Unreleased
 
+- GARM `v0.2.1-nddev.93` retires excess undemanded idle ephemeral JIT
+  registrations through the Actions service RemoveRunner path (204 then 404)
+  after two matching observations. REST `busy` must be explicit false; an
+  omitted field is unknown, not idle. Scale-set statistics of all zeros are
+  not proof that nothing is running. Identity changes, demand, min-idle,
+  bootstrap/active states and JobStillRunningException refuse the delete.
+  Provider delete remains GARM's ordinary absent-runner reconcile. Idle
+  retirement is best-effort: a remote read or delete failure is a classified
+  warning and cannot block absent-runner consolidation. The original
+  consolidation lock lifetime is unchanged: runner Unlock stays deferred until
+  consolidateRunnerState returns. Candidate age and
+  min-idle are decided locally before any new API call. The `.92` patches are
+  unchanged. REST overlay fallback for acknowledged never-started jobs is not
+  in this derivative: JobAssigned does not bind GitHub job ID or run attempt.
+- Ordinary merge in this repository does not require a general CI status
+  check. `Gate` remains truthful advisory evidence.
+
 - Confirm current GitHub scale-set demand before each new JIT registration.
   A retained local assignment requests reconciliation but cannot allocate a
   runner against a fresh zero-demand snapshot. Reads have a five-second

config/example-runner-1.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.92
+  manager_version: v0.2.1-nddev.93
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-2.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.92
+  manager_version: v0.2.1-nddev.93
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-3.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.92
+  manager_version: v0.2.1-nddev.93
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-4.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.92
+  manager_version: v0.2.1-nddev.93
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-services.yaml

@@ -24,7 +24,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.92
+  manager_version: v0.2.1-nddev.93
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/fleet-contract.yaml

@@ -19,7 +19,7 @@ schema_version: 2
 # a runner class withdrawn, a tenancy boundary narrowed, an admission rule
 # changed. Adding a class or a tenant does not bump it -- that only widens what
 # is on offer.
-contract_version: 5
+contract_version: 6
 
 execution:
   worker_kind: incus-container

config/garm-derivative.yaml

@@ -1,6 +1,6 @@
 schema_version: 1
 artifact: garm
-derivative_version: v0.2.1-nddev.92
+derivative_version: v0.2.1-nddev.93
 upstream:
   repository: https://github.com/cloudbase/garm
   release: v0.2.1
@@ -97,6 +97,9 @@ patches:
   - path: third_party/garm/patches/0030-confirm-live-scale-set-demand.patch
     sha256: 102aebd1fb51a7d4bd619fbbb5ab6e3e5706677e35cdbd6c76663640a24f1497
     purpose: Confirm current GitHub scale-set demand with a bounded read before JIT registration, preserving admission limits and queued identities while rejecting stale local-only demand.
+  - path: third_party/garm/patches/0031-retire-excess-idle-online-runners.patch
+    sha256: 62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345
+    purpose: Require Actions RemoveRunner to return 204 and call warnIdleRetirement before the original consolidateRunnerState body so idle-retirement failures cannot block absent-runner consolidation while runner locks stay held until that function returns.
 overlays:
   - path: third_party/garm/overlay/workers/scaleset/queue_intent.go
     sha256: be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7
@@ -116,6 +119,12 @@ overlays:
   - path: third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go
     sha256: 32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8
     purpose: Prove fresh zero demand prevents allocation, stale persisted zero can recover against fresh positive demand, positive observations are not reused, and missing, mismatched or unavailable evidence fails closed.
+  - path: third_party/garm/overlay/workers/scaleset/idle_online_retire.go
+    sha256: a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69
+    purpose: Retire excess undemanded idle ephemeral JIT registrations through Actions RemoveRunner after two matching observations, requiring REST busy to be explicit false and refusing omitted busy, identity change, demand, or JobStillRunningException. Local age and min-idle decide whether any API call happens; unseen observation IDs are pruned.
+  - path: third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go
+    sha256: 94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4
+    purpose: Prove omitted REST busy is unknown, Actions list Busy defaults are untrusted, all-zero scale-set statistics do not prove idle, mixed identity restarts observation, two matching observations are required before RemoveRunner, empty aged-candidate plans make no API call, and helper RemoveRunner 204+404, 409, unexpected 2xx and non-404 read-back outcomes. These are predicate and fake-client helpers, not Worker/DB/HTTP integration.
 build:
   container_image: docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36
   go_version: go1.26.6
@@ -130,7 +139,7 @@ build:
     - sqlite_omit_load_extension
   reproducible_rebuilds: 2
   maximum_required_glibc: "2.34"
-  binary_sha256: c112777344debf7e9c0b82524b6de381872ff6576e700cc4c4469b136770b560
+  binary_sha256: 8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841
 runtime_contract:
   queue_intent_schema_version: 6
   event_driven_scale_set_wake: true
@@ -168,6 +177,7 @@ runtime_contract:
   authoritative_repository_binding: true
   authoritative_running_repository_binding: true
   authoritative_idle_offline_runner_reaping: true
+  authoritative_idle_online_runner_retirement: true
   authoritative_access_refusal_backoff_seconds: 900
   job_started_runner_identity: true
   job_available_correlation_binding: true

docs/runbooks/fleet-alerts.md

@@ -9,6 +9,22 @@ use the exact journal identity and its first-queued/state-entry timestamps to
 reconstruct the wait. A lower aggregate maximum can mean that one job advanced
 or was cancelled while other work remains stalled. Check its actual outcome.
 
+Idle online JIT registrations with local pending/idle status are not reaped
+by the `.92` offline-only timeout path. Excess undemanded idle capacity is
+retired only after two matching observations, an exact identity match, age
+of at least thirty minutes, REST `busy` explicitly false, and Actions
+RemoveRunner returning 204 with a 404 read-back. An omitted REST `busy`
+field is unknown, not idle. Scale-set statistics of all zeros are not
+proof that nothing is running; a later REST `busy=true` must refuse the
+delete. GARM then deletes the provider instance only after the
+registration is absent. Never use the public GitHub REST force-delete.
+
+`lifecycle_assigned_stall` is the assigned *state clock*, not
+`FirstQueuedAt`. Assigned TTL is 600 seconds; a lower aggregate maximum
+can mean the oldest assigned job started, cancelled, or demoted while a
+queued waiter with an hours-old `FirstQueuedAt` remains. Do not shorten
+that alert to hide age.
+
 A REST workflow job can remain queued while the scale-set API reports no jobs
 assigned to that scale set. Preserve both observations. GARM confirms the
 scale-set API's current `statistics.TotalAssignedJobs` before creating a new
@@ -17,6 +33,8 @@ old local `JobAssigned` record is a reason to reconcile; it is not independent
 proof of dispatchable demand. Conversely, an old persisted zero must not block
 an authoritative read that could show new demand. GitHub documents this
 distinction in the [scale-set client autoscaling contract](https://github.com/actions/scaleset#autoscaling).
+Idle-online retirement is best-effort and cannot block absent-runner
+consolidation. It does not repair a lost upstream assignment.
 
 Compare a start with its completion in the same message batch when diagnosing
 fast jobs. Completion is terminal even if no intermediate running observation

internal/fleetcontract/contract.go

@@ -366,8 +366,8 @@ func loadMerge(path string) (Merge, error) {
 	if err := yaml.Unmarshal(raw, &declared); err != nil {
 		return Merge{}, fmt.Errorf("parse branch protection declaration: %w", err)
 	}
-	if declared.Branch == "" || len(declared.RequiredStatusChecks.Contexts) == 0 {
-		return Merge{}, fmt.Errorf("branch protection declaration names no branch or no required context")
+	if declared.Branch == "" {
+		return Merge{}, fmt.Errorf("branch protection declaration names no branch")
 	}
 	contexts := slices.Clone(declared.RequiredStatusChecks.Contexts)
 	slices.Sort(contexts)

internal/fleetcontract/contract_test.go

@@ -29,10 +29,10 @@ func TestPublicExampleContractBuildsWithoutEstateAccess(t *testing.T) {
 		t.Fatal(err)
 	}
 	if contract.Repository != "NDDev-OpenNetwork/github-actions" || len(contract.RunnerClasses) == 0 ||
-		len(contract.Tenants) == 0 || len(contract.Merge.RequiredContexts) != 1 || contract.Merge.RequiredContexts[0] != "Gate" {
+		len(contract.Tenants) == 0 || len(contract.Merge.RequiredContexts) != 0 {
 		t.Fatalf("public contract = %#v", contract)
 	}
-	if contract.SchemaVersion != 2 || contract.ContractVersion != 5 || contract.Execution.WorkerKind != "incus-container" ||
+	if contract.SchemaVersion != 2 || contract.ContractVersion != 6 || contract.Execution.WorkerKind != "incus-container" ||
 		!contract.Execution.Ephemeral || contract.Execution.JobsPerWorker != 1 || !contract.ResourceSemantics.HardMemoryExcludesEmergencySwap ||
 		contract.ResourceSemantics.EmergencySwapSchedulable || contract.ResourceSemantics.CPUMode != "weighted-overcommit" {
 		t.Fatalf("contract v2 semantics = %#v", contract)

internal/garmderivative/script.go

@@ -117,6 +117,7 @@ var runtimeContractFields = []string{
 	"authoritative_repository_binding",
 	"authoritative_running_repository_binding",
 	"authoritative_idle_offline_runner_reaping",
+	"authoritative_idle_online_runner_retirement",
 	"authoritative_access_refusal_backoff_seconds",
 	"job_started_runner_identity",
 	"job_available_correlation_binding",

internal/garmderivative/types.go

@@ -100,6 +100,7 @@ type RuntimeContract struct {
 	AuthoritativeRepositoryBinding           bool   `json:"authoritative_repository_binding" yaml:"authoritative_repository_binding"`
 	AuthoritativeRunningRepositoryBinding    bool   `json:"authoritative_running_repository_binding" yaml:"authoritative_running_repository_binding"`
 	AuthoritativeIdleOfflineRunnerReaping    bool   `json:"authoritative_idle_offline_runner_reaping" yaml:"authoritative_idle_offline_runner_reaping"`
+	AuthoritativeIdleOnlineRunnerRetirement  bool   `json:"authoritative_idle_online_runner_retirement" yaml:"authoritative_idle_online_runner_retirement"`
 	AuthoritativeAccessRefusalBackoffSeconds int    `json:"authoritative_access_refusal_backoff_seconds" yaml:"authoritative_access_refusal_backoff_seconds"`
 	JobStartedRunnerIdentity                 bool   `json:"job_started_runner_identity" yaml:"job_started_runner_identity"`
 	JobAvailableCorrelationBinding           bool   `json:"job_available_correlation_binding" yaml:"job_available_correlation_binding"`

scripts/build-garm-nddev.sh

@@ -19,7 +19,7 @@ set -Eeuo pipefail
 # Every value below is the manifest's. Editing one here detaches the build
 # from the provenance it is reviewed against, which is why the region is
 # regenerated and compared rather than maintained.
-readonly derivative_version="v0.2.1-nddev.92"
+readonly derivative_version="v0.2.1-nddev.93"
 readonly upstream_repository="https://github.com/cloudbase/garm"
 readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684"
 readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36"
@@ -32,7 +32,7 @@ readonly build_module_mode="vendor"
 readonly build_tags="osusergo,netgo,sqlite_omit_load_extension"
 readonly build_reproducible_rebuilds="2"
 readonly build_maximum_required_glibc="2.34"
-readonly expected_binary_sha256="c112777344debf7e9c0b82524b6de381872ff6576e700cc4c4469b136770b560"
+readonly expected_binary_sha256="8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841"
 readonly patch_paths=(
   "third_party/garm/patches/0001-event-driven-reconciliation.patch"
   "third_party/garm/patches/0002-central-queue-admission.patch"
@@ -64,6 +64,7 @@ readonly patch_paths=(
   "third_party/garm/patches/0028-terminal-workflow-run-reconciliation.patch"
   "third_party/garm/patches/0029-scale-up-from-admitted-intent.patch"
   "third_party/garm/patches/0030-confirm-live-scale-set-demand.patch"
+  "third_party/garm/patches/0031-retire-excess-idle-online-runners.patch"
 )
 readonly patch_sha256s=(
   "2f0571f141e7388d6ea0cb0341549ba5bf5dab26d0006382a71b76655e272d34"
@@ -96,6 +97,7 @@ readonly patch_sha256s=(
   "3bc2722e87f1721c6094130d0f07310a796bf8074504ef121acac8ca7402887c"
   "e63c57a3c0a9d492ba45bc35f02e1f9abbc84e7e7923717b3be6ff45206f8a0d"
   "102aebd1fb51a7d4bd619fbbb5ab6e3e5706677e35cdbd6c76663640a24f1497"
+  "62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345"
 )
 readonly overlay_paths=(
   "third_party/garm/overlay/workers/scaleset/queue_intent.go"
@@ -104,6 +106,8 @@ readonly overlay_paths=(
   "third_party/garm/overlay/workers/provider/nddev_create_retry_test.go"
   "third_party/garm/overlay/workers/scaleset/confirmed_demand.go"
   "third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go"
+  "third_party/garm/overlay/workers/scaleset/idle_online_retire.go"
+  "third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go"
 )
 readonly overlay_sha256s=(
   "be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7"
@@ -112,6 +116,8 @@ readonly overlay_sha256s=(
   "c52b783f1b420a3bb15fdc3ca7a445e295251e82e64a85f4d2b41de474546eff"
   "5c289dc30a52e5329e3d4887660629e7d12f855b5c8204d2d99e68c003f39458"
   "32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8"
+  "a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69"
+  "94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4"
 )
 readonly overlay_targets=(
   "workers/scaleset/queue_intent.go"
@@ -120,6 +126,8 @@ readonly overlay_targets=(
   "workers/provider/nddev_create_retry_test.go"
   "workers/scaleset/confirmed_demand.go"
   "workers/scaleset/confirmed_demand_test.go"
+  "workers/scaleset/idle_online_retire.go"
+  "workers/scaleset/idle_online_retire_test.go"
 )
 # END GENERATED REGION