fix(garm): exact identity and dispatch occupancy for v0.2.1-nddev.94

CHANGELOG.md

@@ -2,6 +2,30 @@
 
 ## Unreleased
 
+- GARM `v0.2.1-nddev.94` binds stale scale-set job mutation to exact identity:
+  check-run `external_id`, `workflow_job.check_run_url`, repository, exact
+  attempt, numeric job ID and source SHA. A job name is never terminal proof.
+  Missing fields, incomplete pagination, omitted or changing page totals,
+  another attempt, a non-Actions producer and no exact match retain the
+  intent. Scaling uses the latest MESSAGE `statistics.TotalAssignedJobs`;
+  idle retirement requires a recent MESSAGE with `messageID > 0` and
+  re-checks that observation before RemoveRunner. Session-create zeros and
+  202/nil long-polls are not idle evidence. Start failure deletes the new
+  message session and cancels the listener context. Listener JobCompleted
+  with an empty runner name ends a delivery reservation and is not a REST
+  workflow-job terminal; REST still-queued exact identity can clear that
+  tombstone. A later same-run/name GUID is not aliased onto it. JobStarted
+  of another GUID does not delete or rename an assigned waiter or copy its
+  FIFO clock. A request-less JobAssigned yields occupancy when a different
+  GUID becomes JobAvailable, including while that reservation is still
+  unexpired; the original waiter and FIFO stay in the journal. Same-GUID
+  JobAvailable keeps occupancy. A replayed MESSAGE with the same session and
+  messageID does not refresh idle-retirement freshness. A late message from a
+  replaced session does not overwrite current demand.
+  `golang.org/x/text` is v0.39.0. The `.92` and `.93`
+  patches are unchanged. This is a source/artifact candidate, not a fleet
+  rollout.
+
 - GARM `v0.2.1-nddev.93` retires excess undemanded idle ephemeral JIT
   registrations through the Actions service RemoveRunner path (204 then 404)
   after two matching observations. REST `busy` must be explicit false; an

config/example-runner-1.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.93
+  manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-2.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.93
+  manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-3.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.93
+  manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-runner-4.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.93
+  manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/example-services.yaml

@@ -24,7 +24,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.93
+  manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.129

config/garm-derivative.yaml

@@ -1,6 +1,6 @@
 schema_version: 1
 artifact: garm
-derivative_version: v0.2.1-nddev.93
+derivative_version: v0.2.1-nddev.94
 upstream:
   repository: https://github.com/cloudbase/garm
   release: v0.2.1
@@ -100,31 +100,40 @@ patches:
   - path: third_party/garm/patches/0031-retire-excess-idle-online-runners.patch
     sha256: 62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345
     purpose: Require Actions RemoveRunner to return 204 and call warnIdleRetirement before the original consolidateRunnerState body so idle-retirement failures cannot block absent-runner consolidation while runner locks stay held until that function returns.
+  - path: third_party/garm/patches/0032-exact-scale-set-job-identity.patch
+    sha256: 396eea6c1ee9250843ed8002e57658c8a1a6d853423ecba7ed5f85442ec59d9c
+    purpose: Bind scale-set stale-job mutation to check-run external_id, workflow_job.check_run_url path ID, repository, exact attempt, numeric job ID and source SHA. A name is a search hint only. Missing fields, omitted or changing page totals, incomplete pagination, wrong URL, wrong scale-set, non-Actions producer, access refusal and no exact match retain the intent.
+  - path: third_party/garm/patches/0033-live-message-demand-authority.patch
+    sha256: e2a9bb7cc787a3c1dc7bfe4c8e1af458cff8ed77ddd8d5b135c0402bef4a5dbb
+    purpose: Treat statistics.TotalAssignedJobs from the current MESSAGE session as scaling authority. Idle retirement requires a recent MESSAGE with messageID>0. A replayed MESSAGE with the same session and messageID does not refresh observedAt. A late message from a replaced session does not replace current demand. Session-create zeros and 202/nil long-polls are not idle evidence. Start failure deletes the new session and cancels the listener context. Accessors are mux-protected.
+  - path: third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch
+    sha256: 2d23c290fe462553607739afeccca2fe1dcc13d2352e6324464f35fa3ffb6bc2
+    purpose: Upgrade the one source-reachable advisory, golang.org/x/text, to v0.39.0. x/mod v0.37.0 and x/sync v0.21.0 are required transitives of that release. OpenPGP is not imported.
 overlays:
   - path: third_party/garm/overlay/workers/scaleset/queue_intent.go
-    sha256: be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7
-    purpose: Fsync-backed central queue admission that excludes terminal capacity, gives completion precedence in fast start/complete batches, requests fresh demand when persisted desired is zero, demotes an expired never-started assigned waiter to queued instead of deleting it, refreshes queued TTL up to the execution horizon, and FIFO-orders by FirstQueuedAt so a rewritten QueueTime cannot starve a GitHub-queued job behind a later UUID on the same label.
+    sha256: 44197acff7b8643827ab02637d60d2bbb06470313665a94e27c419291b2428ac
+    purpose: Fsync-backed central queue admission that excludes terminal capacity, gives completion precedence in fast start/complete batches, requests fresh demand when persisted desired is zero, demotes an expired never-started assigned waiter to queued instead of deleting it, refreshes queued TTL up to the execution horizon, FIFO-orders by FirstQueuedAt, treats listener JobCompleted as a delivery-reservation end rather than a REST workflow-job terminal, does not alias a later same-run/name GUID onto that waiter, does not delete or rename an assigned waiter because an unrelated JobStarted arrived, and yields request-less occupancy when a dispatchable JobAvailable for another GUID is recorded.
   - path: third_party/garm/overlay/workers/scaleset/queue_intent_test.go
-    sha256: e4680119fd004f56db9801583de3781e70597b1a0f7c79eee9edd7cb95d80cdf
-    purpose: Prove completed batched execution cannot retain running capacity, an admitted waiter requests reconciliation when persisted desired is zero, expired never-started assigned waiters keep FIFO over later JobAssigned tokens, queued waiters refresh until the execution horizon then drop, FirstQueuedAt wins when QueueTime moves, plus lineage transfer, bootstrap, fairness, width and acknowledgement coverage.
+    sha256: 5678a469b4fac0270fce611b1e3d2f7b0a1bc4785d7a358c0a232f688d6b4fc7
+    purpose: Prove completed batched execution cannot retain running capacity, an admitted waiter requests reconciliation when persisted desired is zero, expired never-started assigned waiters keep FIFO over later JobAssigned tokens, queued waiters refresh until the execution horizon then drop, FirstQueuedAt wins when QueueTime moves, a replacement GUID does not inherit a completed waiter's queue clock, REST still-queued identity clears a listener delivery tombstone, unrelated JobStarted does not consume another GUID's assigned waiter or FIFO clock, a dispatchable JobAvailable takes occupancy from a request-less reservation without deleting it including while that reservation is still unexpired, same-GUID JobAvailable keeps occupancy, plus bootstrap, fairness, width and acknowledgement coverage.
   - path: third_party/garm/overlay/workers/provider/nddev_create_retry.go
     sha256: 2fd202d890088680ef9257e0a3366855b6e4d216c41a7a8936d7866998021219
     purpose: Fsync-backed schema-v2 instance-to-intent reservations that bind a pre-job create to a queued non-terminal waiter when no assigned intent exists yet, skip terminal lineage from active inventory, and treat job-keyed capacity refusals as cluster backpressure that never opens a 24-hour circuit.
   - path: third_party/garm/overlay/workers/provider/nddev_create_retry_test.go
     sha256: c52b783f1b420a3bb15fdc3ca7a445e295251e82e64a85f4d2b41de474546eff
     purpose: Prove a pre-job create binds the live queued waiter rather than terminal lineage, unique-owner reconstruction, tenant-qualified shared-capacity ownership, selective terminal pruning, blocked-intent preservation, and fail-closed behavior when every exact intent is blocked.
   - path: third_party/garm/overlay/workers/scaleset/confirmed_demand.go
-    sha256: 5c289dc30a52e5329e3d4887660629e7d12f855b5c8204d2d99e68c003f39458
-    purpose: Bound each new JIT registration by current GitHub demand with a five-second request deadline and thirty-second negative/error backoff; preserve existing jobs and workers.
+    sha256: 8a99d172b4aaf7f113081c776bbde471db6f85420bf5eca5a856a40c4e59606b
+    purpose: Bound each new JIT registration by the latest live MESSAGE assigned counter from the current listener session. Idle retirement additionally requires a recent MESSAGE with messageID>0. Unknown, invalid or unobserved demand fails closed. A 202/nil long-poll is not a heartbeat and does not expire legitimate scaling.
   - path: third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go
-    sha256: 32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8
-    purpose: Prove fresh zero demand prevents allocation, stale persisted zero can recover against fresh positive demand, positive observations are not reused, and missing, mismatched or unavailable evidence fails closed.
+    sha256: 453e6252ab13f29feff0b1fbae6c418e081bb89222accf1551c90bf7eac12a3f
+    purpose: Prove unknown or unobserved demand cannot allocate, a live session zero does not create, a live positive count can, admission and max runners still cap the target, an invalid count fails closed, session-create zeros are not idle-fresh, a changed generation cannot confirm idle removal, and JobStarted demand cannot confirm idle removal. Gate unit tests, not Worker/DB/HTTP integration.
   - path: third_party/garm/overlay/workers/scaleset/idle_online_retire.go
-    sha256: a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69
-    purpose: Retire excess undemanded idle ephemeral JIT registrations through Actions RemoveRunner after two matching observations, requiring REST busy to be explicit false and refusing omitted busy, identity change, demand, or JobStillRunningException. Local age and min-idle decide whether any API call happens; unseen observation IDs are pruned.
+    sha256: c77cc8fff92b5b23eb4691d24acdd58548f21cce065f9837ad6f2f70fd207eb4
+    purpose: Retire excess undemanded idle ephemeral JIT registrations through Actions RemoveRunner after two matching observations, requiring REST busy to be explicit false, a recent MESSAGE assigned-zero, and the same observation immediately before removal. Runner locks are released before the next candidate. Refuses omitted busy, identity change, unknown live message demand, live assigned demand, or JobStillRunningException.
   - path: third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go
-    sha256: 94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4
-    purpose: Prove omitted REST busy is unknown, Actions list Busy defaults are untrusted, all-zero scale-set statistics do not prove idle, mixed identity restarts observation, two matching observations are required before RemoveRunner, empty aged-candidate plans make no API call, and helper RemoveRunner 204+404, 409, unexpected 2xx and non-404 read-back outcomes. These are predicate and fake-client helpers, not Worker/DB/HTTP integration.
+    sha256: d8dbd3b5d3ca0c89cc8f869e27e1a074166be705a1b65e76b70e2f6cdf92d7dc
+    purpose: Prove omitted REST busy is unknown, Actions list Busy defaults are untrusted, unknown message demand cannot retire idle runners, all-zero live statistics do not prove demand by themselves, mixed identity restarts observation, two matching observations are required before RemoveRunner, empty aged-candidate plans make no API call, and helper RemoveRunner 204+404, 409, unexpected 2xx and non-404 read-back outcomes. These are predicate and fake-client helpers, not Worker/DB/HTTP integration.
 build:
   container_image: docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36
   go_version: go1.26.6
@@ -139,7 +148,7 @@ build:
     - sqlite_omit_load_extension
   reproducible_rebuilds: 2
   maximum_required_glibc: "2.34"
-  binary_sha256: 8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841
+  binary_sha256: 44721568b2f8839e643a1976414de56a41e453854b5f41467a2cf2775b99bdad
 runtime_contract:
   queue_intent_schema_version: 6
   event_driven_scale_set_wake: true

scripts/build-garm-nddev.sh

@@ -19,7 +19,7 @@ set -Eeuo pipefail
 # Every value below is the manifest's. Editing one here detaches the build
 # from the provenance it is reviewed against, which is why the region is
 # regenerated and compared rather than maintained.
-readonly derivative_version="v0.2.1-nddev.93"
+readonly derivative_version="v0.2.1-nddev.94"
 readonly upstream_repository="https://github.com/cloudbase/garm"
 readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684"
 readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36"
@@ -32,7 +32,7 @@ readonly build_module_mode="vendor"
 readonly build_tags="osusergo,netgo,sqlite_omit_load_extension"
 readonly build_reproducible_rebuilds="2"
 readonly build_maximum_required_glibc="2.34"
-readonly expected_binary_sha256="8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841"
+readonly expected_binary_sha256="44721568b2f8839e643a1976414de56a41e453854b5f41467a2cf2775b99bdad"
 readonly patch_paths=(
   "third_party/garm/patches/0001-event-driven-reconciliation.patch"
   "third_party/garm/patches/0002-central-queue-admission.patch"
@@ -65,6 +65,9 @@ readonly patch_paths=(
   "third_party/garm/patches/0029-scale-up-from-admitted-intent.patch"
   "third_party/garm/patches/0030-confirm-live-scale-set-demand.patch"
   "third_party/garm/patches/0031-retire-excess-idle-online-runners.patch"
+  "third_party/garm/patches/0032-exact-scale-set-job-identity.patch"
+  "third_party/garm/patches/0033-live-message-demand-authority.patch"
+  "third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch"
 )
 readonly patch_sha256s=(
   "2f0571f141e7388d6ea0cb0341549ba5bf5dab26d0006382a71b76655e272d34"
@@ -98,6 +101,9 @@ readonly patch_sha256s=(
   "e63c57a3c0a9d492ba45bc35f02e1f9abbc84e7e7923717b3be6ff45206f8a0d"
   "102aebd1fb51a7d4bd619fbbb5ab6e3e5706677e35cdbd6c76663640a24f1497"
   "62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345"
+  "396eea6c1ee9250843ed8002e57658c8a1a6d853423ecba7ed5f85442ec59d9c"
+  "e2a9bb7cc787a3c1dc7bfe4c8e1af458cff8ed77ddd8d5b135c0402bef4a5dbb"
+  "2d23c290fe462553607739afeccca2fe1dcc13d2352e6324464f35fa3ffb6bc2"
 )
 readonly overlay_paths=(
   "third_party/garm/overlay/workers/scaleset/queue_intent.go"
@@ -110,14 +116,14 @@ readonly overlay_paths=(
   "third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go"
 )
 readonly overlay_sha256s=(
-  "be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7"
-  "e4680119fd004f56db9801583de3781e70597b1a0f7c79eee9edd7cb95d80cdf"
+  "44197acff7b8643827ab02637d60d2bbb06470313665a94e27c419291b2428ac"
+  "5678a469b4fac0270fce611b1e3d2f7b0a1bc4785d7a358c0a232f688d6b4fc7"
   "2fd202d890088680ef9257e0a3366855b6e4d216c41a7a8936d7866998021219"
   "c52b783f1b420a3bb15fdc3ca7a445e295251e82e64a85f4d2b41de474546eff"
-  "5c289dc30a52e5329e3d4887660629e7d12f855b5c8204d2d99e68c003f39458"
-  "32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8"
-  "a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69"
-  "94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4"
+  "8a99d172b4aaf7f113081c776bbde471db6f85420bf5eca5a856a40c4e59606b"
+  "453e6252ab13f29feff0b1fbae6c418e081bb89222accf1551c90bf7eac12a3f"
+  "c77cc8fff92b5b23eb4691d24acdd58548f21cce065f9837ad6f2f70fd207eb4"
+  "d8dbd3b5d3ca0c89cc8f869e27e1a074166be705a1b65e76b70e2f6cdf92d7dc"
 )
 readonly overlay_targets=(
   "workers/scaleset/queue_intent.go"
@@ -177,12 +183,35 @@ for index in "${!overlay_paths[@]}"; do
 done
 
 work_dir=$(mktemp -d)
+cidfile=""
+container_pid=""
 cleanup() {
+  if [[ -n "${container_pid}" ]]; then
+    if [[ -n "${cidfile}" && -f "${cidfile}" ]]; then
+      cid=$(cat "${cidfile}" 2>/dev/null || true)
+      if [[ -n "${cid}" ]]; then
+        "${container_engine}" stop --time 20 "${cid}" >/dev/null 2>&1 || true
+      fi
+    fi
+    wait "${container_pid}" 2>/dev/null || true
+    container_pid=""
+  fi
+  if [[ -n "${cidfile}" && -f "${cidfile}" ]]; then
+    cid=$(cat "${cidfile}" 2>/dev/null || true)
+    if [[ -n "${cid}" ]]; then
+      "${container_engine}" wait "${cid}" >/dev/null 2>&1 || true
+      "${container_engine}" rm -f "${cid}" >/dev/null 2>&1 || true
+    fi
+    rm -f -- "${cidfile}"
+    cidfile=""
+  fi
   if [[ -n "${work_dir:-}" && "${work_dir}" == /tmp/* && -d "${work_dir}" ]]; then
     rm -rf -- "${work_dir}"
   fi
 }
 trap cleanup EXIT
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
 
 source_dir="${work_dir}/source"
 artifact_dir="${work_dir}/artifacts"
@@ -216,8 +245,13 @@ git -C "${source_dir}" diff --check
 # region could not reach and the contract test could not see. The derivative
 # version was exactly that -- compiled into the binary from a literal that
 # nothing compared against the manifest.
+# Do not use --rm: EXIT must stop and wait for this container before deleting
+# the bind-mounted source. Killing the launcher otherwise unmounts /src under
+# a still-running compile. --cidfile requires a path that does not already exist.
 # shellcheck disable=SC2016
-"${container_engine}" run --rm \
+cidfile="${work_dir}/container.id"
+"${container_engine}" run \
+  --cidfile="${cidfile}" \
   --network "${build_network}" \
   --mount "type=bind,src=${source_dir},dst=/src,readonly" \
   --mount "type=bind,src=${artifact_dir},dst=/out" \
@@ -260,7 +294,33 @@ git -C "${source_dir}" diff --check
       build "/out/garm.${attempt}"
       cmp /out/garm.1 "/out/garm.${attempt}"
     done
-  '
+  ' &
+container_pid=$!
+container_status=0
+wait "${container_pid}" || container_status=$?
+container_pid=""
+if [[ ! -f "${cidfile}" ]]; then
+  echo "GARM build container identity is missing" >&2
+  if [[ "${container_status}" -ne 0 ]]; then
+    exit "${container_status}"
+  fi
+  exit 1
+fi
+container_id=$(cat "${cidfile}")
+if [[ -z "${container_id}" ]]; then
+  echo "GARM build container identity is empty" >&2
+  exit 1
+fi
+container_exit=$("${container_engine}" inspect -f '{{.State.ExitCode}}' "${container_id}")
+"${container_engine}" rm -f "${container_id}" >/dev/null
+rm -f -- "${cidfile}"
+cidfile=""
+if [[ "${container_status}" -ne 0 ]]; then
+  exit "${container_status}"
+fi
+if [[ "${container_exit}" != "0" ]]; then
+  exit "${container_exit}"
+fi
 
 first_sha256=$(sha256sum "${artifact_dir}/garm.1" | awk '{print $1}')
 for attempt in $(seq 2 "${build_reproducible_rebuilds}"); do