diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5153..0768065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ ## Unreleased +- GARM `v0.2.1-nddev.94` binds stale scale-set job mutation to exact identity: + check-run `external_id`, `workflow_job.check_run_url`, repository, exact + attempt, numeric job ID and source SHA. A job name is never terminal proof. + Missing fields, incomplete pagination, omitted or changing page totals, + another attempt, a non-Actions producer and no exact match retain the + intent. Scaling uses the latest MESSAGE `statistics.TotalAssignedJobs`; + idle retirement requires a recent MESSAGE with `messageID > 0` and + re-checks that observation before RemoveRunner. Session-create zeros and + 202/nil long-polls are not idle evidence. Start failure deletes the new + message session and cancels the listener context. Listener JobCompleted + with an empty runner name ends a delivery reservation and is not a REST + workflow-job terminal; REST still-queued exact identity can clear that + tombstone. A later same-run/name GUID is not aliased onto it. JobStarted + of another GUID does not delete or rename an assigned waiter or copy its + FIFO clock. A request-less JobAssigned yields occupancy when a different + GUID becomes JobAvailable, including while that reservation is still + unexpired; the original waiter and FIFO stay in the journal. Same-GUID + JobAvailable keeps occupancy. A replayed MESSAGE with the same session and + messageID does not refresh idle-retirement freshness. A late message from a + replaced session does not overwrite current demand. + `golang.org/x/text` is v0.39.0. The `.92` and `.93` + patches are unchanged. This is a source/artifact candidate, not a fleet + rollout. + - GARM `v0.2.1-nddev.93` retires excess undemanded idle ephemeral JIT registrations through the Actions service RemoveRunner path (204 then 404) after two matching observations. REST `busy` must be explicit false; an diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index 5b7d3c5..94706ae 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.93 + manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.129 diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index 56caddf..cc2980d 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.93 + manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.129 diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index 25a3251..aec7a55 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.93 + manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.129 diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index f2d755c..654d48e 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.93 + manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.129 diff --git a/config/example-services.yaml b/config/example-services.yaml index 2c4837b..f32addd 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -24,7 +24,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.93 + manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.129 diff --git a/config/garm-derivative.yaml b/config/garm-derivative.yaml index db722c3..cde3db4 100644 --- a/config/garm-derivative.yaml +++ b/config/garm-derivative.yaml @@ -1,6 +1,6 @@ schema_version: 1 artifact: garm -derivative_version: v0.2.1-nddev.93 +derivative_version: v0.2.1-nddev.94 upstream: repository: https://github.com/cloudbase/garm release: v0.2.1 @@ -100,13 +100,22 @@ patches: - path: third_party/garm/patches/0031-retire-excess-idle-online-runners.patch sha256: 62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345 purpose: Require Actions RemoveRunner to return 204 and call warnIdleRetirement before the original consolidateRunnerState body so idle-retirement failures cannot block absent-runner consolidation while runner locks stay held until that function returns. + - path: third_party/garm/patches/0032-exact-scale-set-job-identity.patch + sha256: 396eea6c1ee9250843ed8002e57658c8a1a6d853423ecba7ed5f85442ec59d9c + purpose: Bind scale-set stale-job mutation to check-run external_id, workflow_job.check_run_url path ID, repository, exact attempt, numeric job ID and source SHA. A name is a search hint only. Missing fields, omitted or changing page totals, incomplete pagination, wrong URL, wrong scale-set, non-Actions producer, access refusal and no exact match retain the intent. + - path: third_party/garm/patches/0033-live-message-demand-authority.patch + sha256: e2a9bb7cc787a3c1dc7bfe4c8e1af458cff8ed77ddd8d5b135c0402bef4a5dbb + purpose: Treat statistics.TotalAssignedJobs from the current MESSAGE session as scaling authority. Idle retirement requires a recent MESSAGE with messageID>0. A replayed MESSAGE with the same session and messageID does not refresh observedAt. A late message from a replaced session does not replace current demand. Session-create zeros and 202/nil long-polls are not idle evidence. Start failure deletes the new session and cancels the listener context. Accessors are mux-protected. + - path: third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch + sha256: 2d23c290fe462553607739afeccca2fe1dcc13d2352e6324464f35fa3ffb6bc2 + purpose: Upgrade the one source-reachable advisory, golang.org/x/text, to v0.39.0. x/mod v0.37.0 and x/sync v0.21.0 are required transitives of that release. OpenPGP is not imported. overlays: - path: third_party/garm/overlay/workers/scaleset/queue_intent.go - sha256: be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7 - purpose: Fsync-backed central queue admission that excludes terminal capacity, gives completion precedence in fast start/complete batches, requests fresh demand when persisted desired is zero, demotes an expired never-started assigned waiter to queued instead of deleting it, refreshes queued TTL up to the execution horizon, and FIFO-orders by FirstQueuedAt so a rewritten QueueTime cannot starve a GitHub-queued job behind a later UUID on the same label. + sha256: 44197acff7b8643827ab02637d60d2bbb06470313665a94e27c419291b2428ac + purpose: Fsync-backed central queue admission that excludes terminal capacity, gives completion precedence in fast start/complete batches, requests fresh demand when persisted desired is zero, demotes an expired never-started assigned waiter to queued instead of deleting it, refreshes queued TTL up to the execution horizon, FIFO-orders by FirstQueuedAt, treats listener JobCompleted as a delivery-reservation end rather than a REST workflow-job terminal, does not alias a later same-run/name GUID onto that waiter, does not delete or rename an assigned waiter because an unrelated JobStarted arrived, and yields request-less occupancy when a dispatchable JobAvailable for another GUID is recorded. - path: third_party/garm/overlay/workers/scaleset/queue_intent_test.go - sha256: e4680119fd004f56db9801583de3781e70597b1a0f7c79eee9edd7cb95d80cdf - purpose: Prove completed batched execution cannot retain running capacity, an admitted waiter requests reconciliation when persisted desired is zero, expired never-started assigned waiters keep FIFO over later JobAssigned tokens, queued waiters refresh until the execution horizon then drop, FirstQueuedAt wins when QueueTime moves, plus lineage transfer, bootstrap, fairness, width and acknowledgement coverage. + sha256: 5678a469b4fac0270fce611b1e3d2f7b0a1bc4785d7a358c0a232f688d6b4fc7 + purpose: Prove completed batched execution cannot retain running capacity, an admitted waiter requests reconciliation when persisted desired is zero, expired never-started assigned waiters keep FIFO over later JobAssigned tokens, queued waiters refresh until the execution horizon then drop, FirstQueuedAt wins when QueueTime moves, a replacement GUID does not inherit a completed waiter's queue clock, REST still-queued identity clears a listener delivery tombstone, unrelated JobStarted does not consume another GUID's assigned waiter or FIFO clock, a dispatchable JobAvailable takes occupancy from a request-less reservation without deleting it including while that reservation is still unexpired, same-GUID JobAvailable keeps occupancy, plus bootstrap, fairness, width and acknowledgement coverage. - path: third_party/garm/overlay/workers/provider/nddev_create_retry.go sha256: 2fd202d890088680ef9257e0a3366855b6e4d216c41a7a8936d7866998021219 purpose: Fsync-backed schema-v2 instance-to-intent reservations that bind a pre-job create to a queued non-terminal waiter when no assigned intent exists yet, skip terminal lineage from active inventory, and treat job-keyed capacity refusals as cluster backpressure that never opens a 24-hour circuit. @@ -114,17 +123,17 @@ overlays: sha256: c52b783f1b420a3bb15fdc3ca7a445e295251e82e64a85f4d2b41de474546eff purpose: Prove a pre-job create binds the live queued waiter rather than terminal lineage, unique-owner reconstruction, tenant-qualified shared-capacity ownership, selective terminal pruning, blocked-intent preservation, and fail-closed behavior when every exact intent is blocked. - path: third_party/garm/overlay/workers/scaleset/confirmed_demand.go - sha256: 5c289dc30a52e5329e3d4887660629e7d12f855b5c8204d2d99e68c003f39458 - purpose: Bound each new JIT registration by current GitHub demand with a five-second request deadline and thirty-second negative/error backoff; preserve existing jobs and workers. + sha256: 8a99d172b4aaf7f113081c776bbde471db6f85420bf5eca5a856a40c4e59606b + purpose: Bound each new JIT registration by the latest live MESSAGE assigned counter from the current listener session. Idle retirement additionally requires a recent MESSAGE with messageID>0. Unknown, invalid or unobserved demand fails closed. A 202/nil long-poll is not a heartbeat and does not expire legitimate scaling. - path: third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go - sha256: 32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8 - purpose: Prove fresh zero demand prevents allocation, stale persisted zero can recover against fresh positive demand, positive observations are not reused, and missing, mismatched or unavailable evidence fails closed. + sha256: 453e6252ab13f29feff0b1fbae6c418e081bb89222accf1551c90bf7eac12a3f + purpose: Prove unknown or unobserved demand cannot allocate, a live session zero does not create, a live positive count can, admission and max runners still cap the target, an invalid count fails closed, session-create zeros are not idle-fresh, a changed generation cannot confirm idle removal, and JobStarted demand cannot confirm idle removal. Gate unit tests, not Worker/DB/HTTP integration. - path: third_party/garm/overlay/workers/scaleset/idle_online_retire.go - sha256: a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69 - purpose: Retire excess undemanded idle ephemeral JIT registrations through Actions RemoveRunner after two matching observations, requiring REST busy to be explicit false and refusing omitted busy, identity change, demand, or JobStillRunningException. Local age and min-idle decide whether any API call happens; unseen observation IDs are pruned. + sha256: c77cc8fff92b5b23eb4691d24acdd58548f21cce065f9837ad6f2f70fd207eb4 + purpose: Retire excess undemanded idle ephemeral JIT registrations through Actions RemoveRunner after two matching observations, requiring REST busy to be explicit false, a recent MESSAGE assigned-zero, and the same observation immediately before removal. Runner locks are released before the next candidate. Refuses omitted busy, identity change, unknown live message demand, live assigned demand, or JobStillRunningException. - path: third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go - sha256: 94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4 - purpose: Prove omitted REST busy is unknown, Actions list Busy defaults are untrusted, all-zero scale-set statistics do not prove idle, mixed identity restarts observation, two matching observations are required before RemoveRunner, empty aged-candidate plans make no API call, and helper RemoveRunner 204+404, 409, unexpected 2xx and non-404 read-back outcomes. These are predicate and fake-client helpers, not Worker/DB/HTTP integration. + sha256: d8dbd3b5d3ca0c89cc8f869e27e1a074166be705a1b65e76b70e2f6cdf92d7dc + purpose: Prove omitted REST busy is unknown, Actions list Busy defaults are untrusted, unknown message demand cannot retire idle runners, all-zero live statistics do not prove demand by themselves, mixed identity restarts observation, two matching observations are required before RemoveRunner, empty aged-candidate plans make no API call, and helper RemoveRunner 204+404, 409, unexpected 2xx and non-404 read-back outcomes. These are predicate and fake-client helpers, not Worker/DB/HTTP integration. build: container_image: docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36 go_version: go1.26.6 @@ -139,7 +148,7 @@ build: - sqlite_omit_load_extension reproducible_rebuilds: 2 maximum_required_glibc: "2.34" - binary_sha256: 8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841 + binary_sha256: 44721568b2f8839e643a1976414de56a41e453854b5f41467a2cf2775b99bdad runtime_contract: queue_intent_schema_version: 6 event_driven_scale_set_wake: true diff --git a/scripts/build-garm-nddev.sh b/scripts/build-garm-nddev.sh index 4b9d311..9aedbac 100755 --- a/scripts/build-garm-nddev.sh +++ b/scripts/build-garm-nddev.sh @@ -19,7 +19,7 @@ set -Eeuo pipefail # Every value below is the manifest's. Editing one here detaches the build # from the provenance it is reviewed against, which is why the region is # regenerated and compared rather than maintained. -readonly derivative_version="v0.2.1-nddev.93" +readonly derivative_version="v0.2.1-nddev.94" readonly upstream_repository="https://github.com/cloudbase/garm" readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684" readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36" @@ -32,7 +32,7 @@ readonly build_module_mode="vendor" readonly build_tags="osusergo,netgo,sqlite_omit_load_extension" readonly build_reproducible_rebuilds="2" readonly build_maximum_required_glibc="2.34" -readonly expected_binary_sha256="8f6519500e90ad028c62b7fc3f5c8f2c7784fd82ef653c37c8db9e45afcfa841" +readonly expected_binary_sha256="44721568b2f8839e643a1976414de56a41e453854b5f41467a2cf2775b99bdad" readonly patch_paths=( "third_party/garm/patches/0001-event-driven-reconciliation.patch" "third_party/garm/patches/0002-central-queue-admission.patch" @@ -65,6 +65,9 @@ readonly patch_paths=( "third_party/garm/patches/0029-scale-up-from-admitted-intent.patch" "third_party/garm/patches/0030-confirm-live-scale-set-demand.patch" "third_party/garm/patches/0031-retire-excess-idle-online-runners.patch" + "third_party/garm/patches/0032-exact-scale-set-job-identity.patch" + "third_party/garm/patches/0033-live-message-demand-authority.patch" + "third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch" ) readonly patch_sha256s=( "2f0571f141e7388d6ea0cb0341549ba5bf5dab26d0006382a71b76655e272d34" @@ -98,6 +101,9 @@ readonly patch_sha256s=( "e63c57a3c0a9d492ba45bc35f02e1f9abbc84e7e7923717b3be6ff45206f8a0d" "102aebd1fb51a7d4bd619fbbb5ab6e3e5706677e35cdbd6c76663640a24f1497" "62f8a273e86ed19fe80905ad7205fbf5d6fdefdf1847cc085303c613db3f1345" + "396eea6c1ee9250843ed8002e57658c8a1a6d853423ecba7ed5f85442ec59d9c" + "e2a9bb7cc787a3c1dc7bfe4c8e1af458cff8ed77ddd8d5b135c0402bef4a5dbb" + "2d23c290fe462553607739afeccca2fe1dcc13d2352e6324464f35fa3ffb6bc2" ) readonly overlay_paths=( "third_party/garm/overlay/workers/scaleset/queue_intent.go" @@ -110,14 +116,14 @@ readonly overlay_paths=( "third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go" ) readonly overlay_sha256s=( - "be85eac2126d7723483467958222f401061bd08a68113af61ff4b51208f717d7" - "e4680119fd004f56db9801583de3781e70597b1a0f7c79eee9edd7cb95d80cdf" + "44197acff7b8643827ab02637d60d2bbb06470313665a94e27c419291b2428ac" + "5678a469b4fac0270fce611b1e3d2f7b0a1bc4785d7a358c0a232f688d6b4fc7" "2fd202d890088680ef9257e0a3366855b6e4d216c41a7a8936d7866998021219" "c52b783f1b420a3bb15fdc3ca7a445e295251e82e64a85f4d2b41de474546eff" - "5c289dc30a52e5329e3d4887660629e7d12f855b5c8204d2d99e68c003f39458" - "32f7485aeae73c15b28148d8d244e93a35cd8fb6dccea99a418db890107604d8" - "a7eb84571633d76ce6f3134b362e90b695beb7f4f36c882363bef087ad2d3d69" - "94c4d4ba7368adba4e9620c89b116acccba6c09771413896150c4b0c5379dbc4" + "8a99d172b4aaf7f113081c776bbde471db6f85420bf5eca5a856a40c4e59606b" + "453e6252ab13f29feff0b1fbae6c418e081bb89222accf1551c90bf7eac12a3f" + "c77cc8fff92b5b23eb4691d24acdd58548f21cce065f9837ad6f2f70fd207eb4" + "d8dbd3b5d3ca0c89cc8f869e27e1a074166be705a1b65e76b70e2f6cdf92d7dc" ) readonly overlay_targets=( "workers/scaleset/queue_intent.go" @@ -177,12 +183,35 @@ for index in "${!overlay_paths[@]}"; do done work_dir=$(mktemp -d) +cidfile="" +container_pid="" cleanup() { + if [[ -n "${container_pid}" ]]; then + if [[ -n "${cidfile}" && -f "${cidfile}" ]]; then + cid=$(cat "${cidfile}" 2>/dev/null || true) + if [[ -n "${cid}" ]]; then + "${container_engine}" stop --time 20 "${cid}" >/dev/null 2>&1 || true + fi + fi + wait "${container_pid}" 2>/dev/null || true + container_pid="" + fi + if [[ -n "${cidfile}" && -f "${cidfile}" ]]; then + cid=$(cat "${cidfile}" 2>/dev/null || true) + if [[ -n "${cid}" ]]; then + "${container_engine}" wait "${cid}" >/dev/null 2>&1 || true + "${container_engine}" rm -f "${cid}" >/dev/null 2>&1 || true + fi + rm -f -- "${cidfile}" + cidfile="" + fi if [[ -n "${work_dir:-}" && "${work_dir}" == /tmp/* && -d "${work_dir}" ]]; then rm -rf -- "${work_dir}" fi } trap cleanup EXIT +trap 'cleanup; exit 130' INT +trap 'cleanup; exit 143' TERM source_dir="${work_dir}/source" artifact_dir="${work_dir}/artifacts" @@ -216,8 +245,13 @@ git -C "${source_dir}" diff --check # region could not reach and the contract test could not see. The derivative # version was exactly that -- compiled into the binary from a literal that # nothing compared against the manifest. +# Do not use --rm: EXIT must stop and wait for this container before deleting +# the bind-mounted source. Killing the launcher otherwise unmounts /src under +# a still-running compile. --cidfile requires a path that does not already exist. # shellcheck disable=SC2016 -"${container_engine}" run --rm \ +cidfile="${work_dir}/container.id" +"${container_engine}" run \ + --cidfile="${cidfile}" \ --network "${build_network}" \ --mount "type=bind,src=${source_dir},dst=/src,readonly" \ --mount "type=bind,src=${artifact_dir},dst=/out" \ @@ -260,7 +294,33 @@ git -C "${source_dir}" diff --check build "/out/garm.${attempt}" cmp /out/garm.1 "/out/garm.${attempt}" done - ' + ' & +container_pid=$! +container_status=0 +wait "${container_pid}" || container_status=$? +container_pid="" +if [[ ! -f "${cidfile}" ]]; then + echo "GARM build container identity is missing" >&2 + if [[ "${container_status}" -ne 0 ]]; then + exit "${container_status}" + fi + exit 1 +fi +container_id=$(cat "${cidfile}") +if [[ -z "${container_id}" ]]; then + echo "GARM build container identity is empty" >&2 + exit 1 +fi +container_exit=$("${container_engine}" inspect -f '{{.State.ExitCode}}' "${container_id}") +"${container_engine}" rm -f "${container_id}" >/dev/null +rm -f -- "${cidfile}" +cidfile="" +if [[ "${container_status}" -ne 0 ]]; then + exit "${container_status}" +fi +if [[ "${container_exit}" != "0" ]]; then + exit "${container_exit}" +fi first_sha256=$(sha256sum "${artifact_dir}/garm.1" | awk '{print $1}') for attempt in $(seq 2 "${build_reproducible_rebuilds}"); do diff --git a/third_party/garm/overlay/workers/scaleset/confirmed_demand.go b/third_party/garm/overlay/workers/scaleset/confirmed_demand.go index e041627..2648404 100644 --- a/third_party/garm/overlay/workers/scaleset/confirmed_demand.go +++ b/third_party/garm/overlay/workers/scaleset/confirmed_demand.go @@ -8,52 +8,65 @@ import ( "github.com/cloudbase/garm/params" ) -const ( - demandReadTimeout = 5 * time.Second - demandReadBackoff = 30 * time.Second -) +// liveMessageDemand is the last statistics.TotalAssignedJobs observed from the +// current MESSAGE session. It is not the database DesiredRunnerCount row: +// GORM may skip rewriting an unchanged value, UpdatedAt is not a heartbeat, +// and a zero persisted by a previous session is not a fresh observation. +type liveMessageDemand struct { + sessionID string + assigned int + observed bool + messageID int64 + observedAt time.Time + generation uint64 +} + +const idleMessageDemandFreshFor = 2 * time.Minute -type scaleSetDemandReader interface { - GetRunnerScaleSetByID(context.Context, int) (params.RunnerScaleSet, error) +func (d liveMessageDemand) idleFresh(now time.Time) bool { + if !d.observed || d.sessionID == "" || d.assigned < 0 || d.messageID <= 0 || d.observedAt.IsZero() { + return false + } + return !now.Before(d.observedAt) && now.Sub(d.observedAt) <= idleMessageDemandFreshFor } -// confirmedDemandGate bounds new JIT registrations by GitHub's current demand. -// Retained JobAssigned records preserve audit/FIFO identity, but cannot prove -// that GitHub still assigns that work to this scale set. Conversely, the last -// message's persisted count may be stale-low. Read the authoritative snapshot -// before allocating; keep every existing job and runner untouched on refusal. -// The owning Worker's mutex serializes access to nextRead. -type confirmedDemandGate struct { - nextRead time.Time +// scalingKnown is the latest assigned counter from the current session. It is +// intentionally not age-gated: a long-poll 202/nil is not a heartbeat and must +// not expire legitimate in-progress scaling. Destructive idle retirement uses +// idleFresh instead. +func (d liveMessageDemand) scalingKnown() bool { + return d.observed && d.sessionID != "" } -func (g *confirmedDemandGate) target(ctx context.Context, now time.Time, reader scaleSetDemandReader, scaleSet params.ScaleSet, current, admitted int) (int, error) { - if !scaleSet.Enabled || admitted <= current || admitted < 1 || scaleSet.MaxRunners < 1 { - return current, nil +func (d liveMessageDemand) unchangedIdleZero(previous liveMessageDemand, now time.Time) bool { + if !d.idleFresh(now) || !previous.idleFresh(now) || d.assigned != 0 || previous.assigned != 0 { + return false } - if now.Before(g.nextRead) { + return d.sessionID == previous.sessionID && d.messageID == previous.messageID && d.generation == previous.generation +} + +// confirmedDemandGate bounds new JIT registrations by live MESSAGE statistics +// from the current listener session. Official scale-set scaling uses +// statistics.TotalAssignedJobs from session/message responses, not +// GetRunnerScaleSetByID metadata and not REST queued job counts. +// +// Unsupported freshness contract: a 202/nil long-poll does not carry +// statistics, so silence is not a heartbeat and not a fresh zero. +type confirmedDemandGate struct{} + +func (g *confirmedDemandGate) target(_ context.Context, _ time.Time, demand liveMessageDemand, scaleSet params.ScaleSet, current, admitted int) (int, error) { + if !scaleSet.Enabled || admitted <= current || admitted < 1 || scaleSet.MaxRunners < 1 { return current, nil } - // Failed, missing and zero-demand observations all have bounded read cost. - // A successful positive observation is never cached across a new create. - g.nextRead = now.Add(demandReadBackoff) - readCtx, cancel := context.WithTimeout(ctx, demandReadTimeout) - defer cancel() - remote, err := reader.GetRunnerScaleSetByID(readCtx, scaleSet.ScaleSetID) - if err != nil { - return current, fmt.Errorf("read current scale-set demand: %w", err) - } - if remote.ID != scaleSet.ScaleSetID || remote.Name != scaleSet.Name || remote.Enabled == nil || !*remote.Enabled || remote.Statistics == nil { - return current, fmt.Errorf("current scale-set demand has missing or mismatched identity, enabled state or statistics") + if !demand.scalingKnown() { + return current, fmt.Errorf("current message demand is unknown") } - desired := remote.Statistics.TotalAssignedJobs - if desired < 0 || remote.Statistics.TotalRunningJobs < 0 || remote.Statistics.TotalRunningJobs > desired { - return current, fmt.Errorf("current scale-set demand has invalid job counts") + if demand.assigned < 0 { + return current, fmt.Errorf("current message demand is invalid") } - target := min(admitted, desired, int(scaleSet.MaxRunners)) + target := min(admitted, demand.assigned, int(scaleSet.MaxRunners)) if target <= current { return current, nil } - g.nextRead = time.Time{} return target, nil } diff --git a/third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go b/third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go index dc985bb..f95a24e 100644 --- a/third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go +++ b/third_party/garm/overlay/workers/scaleset/confirmed_demand_test.go @@ -2,122 +2,110 @@ package scaleset import ( "context" - "errors" "testing" "time" "github.com/cloudbase/garm/params" ) -type demandReaderFunc func(context.Context, int) (params.RunnerScaleSet, error) - -func (f demandReaderFunc) GetRunnerScaleSetByID(ctx context.Context, id int) (params.RunnerScaleSet, error) { - return f(ctx, id) -} - -func demandFixture() (params.ScaleSet, params.RunnerScaleSet) { - enabled := true - local := params.ScaleSet{ScaleSetID: 5, Name: "example-integration", Enabled: true, MaxRunners: 8, DesiredRunnerCount: 0} - remote := params.RunnerScaleSet{ID: 5, Name: local.Name, Enabled: &enabled, Statistics: ¶ms.RunnerScaleSetStatistic{TotalAssignedJobs: 3}} - return local, remote -} - -func TestConfirmedDemandRetainedWaiterCannotCreateAgainstFreshZero(t *testing.T) { - local, remote := demandFixture() - remote.Statistics.TotalAssignedJobs = 0 - now := time.Now() +func TestConfirmedDemandUnknownSessionFailsClosed(t *testing.T) { + local := params.ScaleSet{Enabled: true, MaxRunners: 8, DesiredRunnerCount: 0} var gate confirmedDemandGate - reads := 0 - reader := demandReaderFunc(func(ctx context.Context, id int) (params.RunnerScaleSet, error) { - reads++ - if id != local.ScaleSetID { - t.Fatalf("wrong scale-set read: %d", id) - } - deadline, ok := ctx.Deadline() - if !ok || time.Until(deadline) > demandReadTimeout { - t.Fatal("demand read must have a bounded deadline") - } - return remote, nil - }) - for _, elapsed := range []time.Duration{0, time.Second, 5 * time.Second, 29 * time.Second} { - target, err := gate.target(context.Background(), now.Add(elapsed), reader, local, 0, 4) - if err != nil || target != 0 { - t.Fatalf("retained waiters allocated capacity: target=%d err=%v", target, err) - } + target, err := gate.target(context.Background(), time.Now(), liveMessageDemand{}, local, 0, 4) + if err == nil || target != 0 { + t.Fatalf("unknown demand allocated capacity: target=%d err=%v", target, err) } - if reads != 1 { - t.Fatalf("zero-demand read storm: %d reads", reads) - } - remote.Statistics.TotalAssignedJobs = 2 - target, err := gate.target(context.Background(), now.Add(demandReadBackoff), reader, local, 0, 4) - if err != nil || target != 2 || reads != 2 { - t.Fatalf("new positive demand did not reopen: target=%d reads=%d err=%v", target, reads, err) + target, err = gate.target(context.Background(), time.Now(), liveMessageDemand{sessionID: "old", assigned: 0, observed: false}, local, 1, 4) + if err == nil || target != 1 { + t.Fatalf("unobserved stale zero allocated capacity: target=%d err=%v", target, err) } } -func TestConfirmedDemandRefreshesStaleZeroAndDoesNotCachePositive(t *testing.T) { - local, remote := demandFixture() +func TestConfirmedDemandUsesLiveMessageStatistics(t *testing.T) { + now := time.Now() var gate confirmedDemandGate - reads := 0 - reader := demandReaderFunc(func(context.Context, int) (params.RunnerScaleSet, error) { reads++; return remote, nil }) - target, err := gate.target(context.Background(), time.Now(), reader, local, 0, 5) - if err != nil || target != 3 { - t.Fatalf("stale local zero stranded current work: target=%d err=%v", target, err) + local := params.ScaleSet{ScaleSetID: 5, Name: "example-integration", Enabled: true, MaxRunners: 8, DesiredRunnerCount: 9} + target, err := gate.target(context.Background(), now, liveMessageDemand{sessionID: "sess-1", assigned: 0, observed: true}, local, 0, 4) + if err != nil || target != 0 { + t.Fatalf("live zero allocated capacity: target=%d err=%v", target, err) } - remote.Statistics.TotalAssignedJobs = 0 - target, err = gate.target(context.Background(), time.Now(), reader, local, 1, 5) - if err != nil || target != 1 || reads != 2 { - t.Fatalf("stale positive demand permitted another create: target=%d reads=%d err=%v", target, reads, err) + target, err = gate.target(context.Background(), now, liveMessageDemand{sessionID: "sess-1", assigned: 3, observed: true}, local, 0, 4) + if err != nil || target != 3 { + t.Fatalf("live positive demand did not reopen: target=%d err=%v", target, err) } } func TestConfirmedDemandPreservesAdmissionAndMaximum(t *testing.T) { - for _, tc := range []struct{ admitted, maximum, current, want int }{{2, 8, 0, 2}, {5, 1, 0, 1}, {0, 8, 0, 0}, {2, 8, 2, 2}, {4, 8, 5, 5}} { - local, remote := demandFixture() - local.MaxRunners = uint(tc.maximum) + for _, tc := range []struct { + admitted, maximum, current, assigned, want int + }{ + {2, 8, 0, 3, 2}, + {5, 1, 0, 3, 1}, + {0, 8, 0, 3, 0}, + {2, 8, 2, 3, 2}, + {4, 8, 5, 3, 5}, + {4, 8, 0, 0, 0}, + } { + local := params.ScaleSet{Enabled: true, MaxRunners: uint(tc.maximum), DesiredRunnerCount: 99} var gate confirmedDemandGate - target, err := gate.target(context.Background(), time.Now(), demandReaderFunc(func(context.Context, int) (params.RunnerScaleSet, error) { return remote, nil }), local, tc.current, tc.admitted) + demand := liveMessageDemand{sessionID: "sess-1", assigned: tc.assigned, observed: true} + target, err := gate.target(context.Background(), time.Now(), demand, local, tc.current, tc.admitted) if err != nil || target != tc.want { t.Fatalf("%+v: target=%d err=%v", tc, target, err) } } } -func TestConfirmedDemandUnknownOrWrongIdentityFailsClosed(t *testing.T) { - for _, kind := range []string{"read-error", "missing-stats", "wrong-id", "wrong-name", "disabled", "unknown-enabled", "negative", "running-exceeds-assigned"} { - t.Run(kind, func(t *testing.T) { - local, remote := demandFixture() - var readErr error - switch kind { - case "read-error": - readErr = errors.New("unavailable") - case "missing-stats": - remote.Statistics = nil - case "wrong-id": - remote.ID++ - case "wrong-name": - remote.Name = "example-other" - case "disabled": - *remote.Enabled = false - case "unknown-enabled": - remote.Enabled = nil - case "negative": - remote.Statistics.TotalAssignedJobs = -1 - case "running-exceeds-assigned": - remote.Statistics.TotalRunningJobs = 4 - } - var gate confirmedDemandGate - reads := 0 - now := time.Now() - reader := demandReaderFunc(func(context.Context, int) (params.RunnerScaleSet, error) { reads++; return remote, readErr }) - target, err := gate.target(context.Background(), now, reader, local, 1, 4) - if err == nil || target != 1 { - t.Fatalf("unverified demand permitted create: target=%d err=%v", target, err) - } - _, _ = gate.target(context.Background(), now.Add(time.Second), reader, local, 1, 4) - if reads != 1 { - t.Fatal("failed read was not bounded") - } - }) +func TestConfirmedDemandInvalidCountFailsClosed(t *testing.T) { + local := params.ScaleSet{Enabled: true, MaxRunners: 8} + var gate confirmedDemandGate + target, err := gate.target(context.Background(), time.Now(), liveMessageDemand{sessionID: "sess-1", assigned: -1, observed: true}, local, 1, 4) + if err == nil || target != 1 { + t.Fatalf("invalid message demand permitted create: target=%d err=%v", target, err) + } +} + +func TestIdleFreshRequiresRecentMessageNotSessionCreate(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + sessionCreate := liveMessageDemand{sessionID: "sess-1", assigned: 0, observed: true, messageID: 0, observedAt: now} + if sessionCreate.idleFresh(now) { + t.Fatal("session-create zero with messageID 0 must not authorize idle retirement") + } + if !sessionCreate.scalingKnown() { + t.Fatal("session-create counter must remain usable for ordinary scaling") + } + stale := liveMessageDemand{sessionID: "sess-1", assigned: 0, observed: true, messageID: 9, observedAt: now.Add(-5 * time.Minute)} + if stale.idleFresh(now) { + t.Fatal("a five-minute-old zero must not be reused as two fresh idle observations") + } + fresh := liveMessageDemand{sessionID: "sess-1", assigned: 0, observed: true, messageID: 9, observedAt: now.Add(-30 * time.Second)} + if !fresh.idleFresh(now) { + t.Fatal("a recent MESSAGE zero must be usable for idle retirement") + } + later := liveMessageDemand{sessionID: "sess-1", assigned: 0, observed: true, messageID: 10, observedAt: now, generation: 2} + if later.unchangedIdleZero(fresh, now) { + t.Fatal("a changed generation must not count as the same idle observation") + } + if !fresh.unchangedIdleZero(fresh, now) { + t.Fatal("the same fresh zero must hold through removal") + } + started := fresh + started.assigned = 1 + started.messageID = 10 + started.generation = 3 + started.observedAt = now + if started.unchangedIdleZero(fresh, now) { + t.Fatal("JobStarted demand must refuse idle removal") + } +} + +func TestConfirmedDemandSessionCreateZeroDoesNotExpireScaling(t *testing.T) { + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + var gate confirmedDemandGate + local := params.ScaleSet{Enabled: true, MaxRunners: 8} + demand := liveMessageDemand{sessionID: "sess-1", assigned: 3, observed: true, messageID: 0, observedAt: now.Add(-time.Hour)} + target, err := gate.target(context.Background(), now, demand, local, 0, 4) + if err != nil || target != 3 { + t.Fatalf("aged scaling counter must not be expired: target=%d err=%v", target, err) } } diff --git a/third_party/garm/overlay/workers/scaleset/idle_online_retire.go b/third_party/garm/overlay/workers/scaleset/idle_online_retire.go index 173a00f..11c3d75 100644 --- a/third_party/garm/overlay/workers/scaleset/idle_online_retire.go +++ b/third_party/garm/overlay/workers/scaleset/idle_online_retire.go @@ -76,31 +76,35 @@ type idleRetirementDecision struct { } type idleRetirementEvidence struct { - AgentID int64 - Name string - ScaleSetID int - LocalStatus params.RunnerStatus - CreatedAt time.Time - MinIdle uint - IdleCount int - Now time.Time - RESTID int64 - RESTName string - RESTStatus string - RESTBusy *bool - ActionsID int64 - ActionsName string - ActionsScaleSetID int - ActionsEphemeral bool - ActionsEnabled bool - ActionsState string - ActionsStatus string - AssignedJobs int - AcquiredJobs int - AvailableJobs int - RunningJobs int - BusyRunners int - StatisticsPresent bool + AgentID int64 + Name string + ScaleSetID int + LocalStatus params.RunnerStatus + CreatedAt time.Time + MinIdle uint + IdleCount int + Now time.Time + RESTID int64 + RESTName string + RESTStatus string + RESTBusy *bool + ActionsID int64 + ActionsName string + ActionsScaleSetID int + ActionsEphemeral bool + ActionsEnabled bool + ActionsState string + ActionsStatus string + AssignedJobs int + AcquiredJobs int + AvailableJobs int + RunningJobs int + BusyRunners int + StatisticsPresent bool + MessageDemandKnown bool + DemandSessionID string + DemandMessageID int64 + DemandGeneration uint64 } func evaluateIdleRetirement(ev idleRetirementEvidence) idleRetirementDecision { @@ -118,6 +122,9 @@ func evaluateIdleRetirement(ev idleRetirementEvidence) idleRetirementDecision { if ev.IdleCount <= int(ev.MinIdle) { return idleRetirementDecision{Reason: "at-or-below-min-idle"} } + if !ev.MessageDemandKnown { + return idleRetirementDecision{Reason: "message-demand-unknown"} + } if !ev.StatisticsPresent { return idleRetirementDecision{Reason: "scale-set-statistics-absent"} } @@ -354,32 +361,41 @@ func (w *Worker) retireExcessIdleCapacity() error { continue } actionsListBusyUntrusted(actionsRunner.Busy) + nowUTC := now + // Assigned demand for idle retirement comes from a recent MESSAGE + // observation. GetRunnerScaleSetByID statistics remain fail-closed + // corroboration of identity/enabled state, not assigned-demand authority. + demand := w.liveMessageDemand() evidence := idleRetirementEvidence{ - AgentID: runner.AgentID, - Name: runner.Name, - ScaleSetID: w.scaleSet.ScaleSetID, - LocalStatus: runner.RunnerStatus, - CreatedAt: runner.CreatedAt, - MinIdle: w.scaleSet.MinIdleRunners, - IdleCount: idleCount, - Now: now, - RESTID: restRunner.GetID(), - RESTName: restRunner.GetName(), - RESTStatus: restRunner.GetStatus(), - RESTBusy: restRunner.Busy, - ActionsID: actionsRunner.ID, - ActionsName: actionsRunner.Name, - ActionsScaleSetID: actionsRunner.RunnerScaleSetID, - ActionsEphemeral: actionsRunner.Ephemeral, - ActionsEnabled: actionsRunner.Enabled, - ActionsState: actionsRunner.ProvisioningState, - ActionsStatus: actionsStatusString(actionsRunner.Status), - AssignedJobs: remote.Statistics.TotalAssignedJobs, - AcquiredJobs: remote.Statistics.TotalAcquiredJobs, - AvailableJobs: remote.Statistics.TotalAvailableJobs, - RunningJobs: remote.Statistics.TotalRunningJobs, - BusyRunners: remote.Statistics.TotalBusyRunners, - StatisticsPresent: true, + AgentID: runner.AgentID, + Name: runner.Name, + ScaleSetID: w.scaleSet.ScaleSetID, + LocalStatus: runner.RunnerStatus, + CreatedAt: runner.CreatedAt, + MinIdle: w.scaleSet.MinIdleRunners, + IdleCount: idleCount, + Now: now, + RESTID: restRunner.GetID(), + RESTName: restRunner.GetName(), + RESTStatus: restRunner.GetStatus(), + RESTBusy: restRunner.Busy, + ActionsID: actionsRunner.ID, + ActionsName: actionsRunner.Name, + ActionsScaleSetID: actionsRunner.RunnerScaleSetID, + ActionsEphemeral: actionsRunner.Ephemeral, + ActionsEnabled: actionsRunner.Enabled, + ActionsState: actionsRunner.ProvisioningState, + ActionsStatus: actionsStatusString(actionsRunner.Status), + AssignedJobs: demand.assigned, + AcquiredJobs: remote.Statistics.TotalAcquiredJobs, + AvailableJobs: remote.Statistics.TotalAvailableJobs, + RunningJobs: remote.Statistics.TotalRunningJobs, + BusyRunners: remote.Statistics.TotalBusyRunners, + StatisticsPresent: true, + MessageDemandKnown: demand.idleFresh(nowUTC), + DemandSessionID: demand.sessionID, + DemandMessageID: demand.messageID, + DemandGeneration: demand.generation, } decision := w.idleRetire.confirm(evidence) if !decision.Eligible { @@ -393,8 +409,25 @@ func (w *Worker) retireExcessIdleCapacity() error { if ok := locking.TryLock(runner.Name, w.consumerID); !ok { continue } - defer locking.Unlock(runner.Name, false) - if err := w.removeIdleRunnerAfterConfirmation(cli, runner); err != nil { + lockedName := runner.Name + unlockRunner := func() { + if lockedName == "" { + return + } + locking.Unlock(lockedName, false) + lockedName = "" + } + latest := w.liveMessageDemand() + if !latest.unchangedIdleZero(demand, time.Now().UTC()) { + unlockRunner() + w.idleRetire.forget(runner.AgentID) + slog.InfoContext(w.ctx, "idle retirement skipped; live message demand changed before removal", + "runner_name", runner.Name, "agent_id", runner.AgentID) + continue + } + err = w.removeIdleRunnerAfterConfirmation(cli, runner) + unlockRunner() + if err != nil { w.idleRetire.forget(runner.AgentID) return err } diff --git a/third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go b/third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go index ede342f..cf0a3a6 100644 --- a/third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go +++ b/third_party/garm/overlay/workers/scaleset/idle_online_retire_test.go @@ -38,26 +38,27 @@ func TestRestBusyOmittedIsUnknown(t *testing.T) { func baseEvidence(now time.Time) idleRetirementEvidence { busy := false return idleRetirementEvidence{ - AgentID: 42, - Name: "example-runner", - ScaleSetID: 7, - LocalStatus: params.RunnerPending, - CreatedAt: now.Add(-45 * time.Minute), - MinIdle: 0, - IdleCount: 4, - Now: now, - RESTID: 42, - RESTName: "example-runner", - RESTStatus: "online", - RESTBusy: &busy, - ActionsID: 42, - ActionsName: "example-runner", - ActionsScaleSetID: 7, - ActionsEphemeral: true, - ActionsEnabled: true, - ActionsState: "Provisioned", - ActionsStatus: "online", - StatisticsPresent: true, + AgentID: 42, + Name: "example-runner", + ScaleSetID: 7, + LocalStatus: params.RunnerPending, + CreatedAt: now.Add(-45 * time.Minute), + MinIdle: 0, + IdleCount: 4, + Now: now, + RESTID: 42, + RESTName: "example-runner", + RESTStatus: "online", + RESTBusy: &busy, + ActionsID: 42, + ActionsName: "example-runner", + ActionsScaleSetID: 7, + ActionsEphemeral: true, + ActionsEnabled: true, + ActionsState: "Provisioned", + ActionsStatus: "online", + StatisticsPresent: true, + MessageDemandKnown: true, } } @@ -104,6 +105,18 @@ func TestEvaluateIdleRetirementStatsZeroIsNotABlocker(t *testing.T) { } } +func TestEvaluateIdleRetirementUnknownMessageDemandRefuses(t *testing.T) { + t.Parallel() + now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) + ev := baseEvidence(now) + ev.MessageDemandKnown = false + ev.AssignedJobs = 0 + got := evaluateIdleRetirement(ev) + if got.Eligible || got.Reason != "message-demand-unknown" { + t.Fatalf("stale or disconnected zero must not retire idle runners: %#v", got) + } +} + func TestEvaluateIdleRetirementRefusesPositiveDemand(t *testing.T) { t.Parallel() now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) diff --git a/third_party/garm/overlay/workers/scaleset/queue_intent.go b/third_party/garm/overlay/workers/scaleset/queue_intent.go index 4df4cf0..ece4289 100644 --- a/third_party/garm/overlay/workers/scaleset/queue_intent.go +++ b/third_party/garm/overlay/workers/scaleset/queue_intent.go @@ -243,12 +243,22 @@ func (c *queueIntentCoordinator) EnsureAuthoritative(scaleSet params.ScaleSet, e if queueTime.IsZero() || queueTime.After(now) { queueTime = now } + clearedDeliveryTombstone := false + if _, terminal := journal.TerminalJobs[job.ScaleSetJobID]; terminal { + // REST still-queued identity outranks a listener delivery tombstone. + delete(journal.TerminalJobs, job.ScaleSetJobID) + clearedDeliveryTombstone = true + changed = true + } if existing, exists := journal.Intents[key]; exists { if existing.ScaleSetName != scaleSet.Name || existing.Owner != entity.Owner || !queueIntentRepositoryCompatible(existing, queueIntent{Owner: entity.Owner, Repository: repository}) { return fmt.Errorf("authoritative queued job changed immutable queue identity") } if queueIntentRepositoryBound(existing) { + if clearedDeliveryTombstone { + admitQueuedToBudget(journal, config, now) + } return nil } // Direct JIT normally goes from JobAssigned straight to JobStarted, @@ -271,6 +281,9 @@ func (c *queueIntentCoordinator) EnsureAuthoritative(scaleSet params.ScaleSet, e existing.UpdatedAt = now journal.Intents[key] = existing ensureRepositoryState(journal, config, repository) + if clearedDeliveryTombstone { + admitQueuedToBudget(journal, config, now) + } changed = true return nil } @@ -630,11 +643,13 @@ func (c *queueIntentCoordinator) ObserveLifecycle(scaleSet params.ScaleSet, enti key := queueIntentKey(int64(scaleSet.ScaleSetID), job.JobID) intent, exists := journal.Intents[key] _, startedInBatch := startedKeys[key] - // GitHub may retire a capacity waiter before it ever emits - // JobAvailable, then immediately assign the same workflow job under a - // new UUID. Retain that unstarted intent as a terminal lineage marker; - // it is excluded from admission below and lets the replacement inherit - // the original queue time instead of starving at the tail forever. + // Listener JobCompleted with no runner ends this GUID's scale-set + // delivery reservation. It is not a REST workflow-job outcome: the + // GitHub job may remain queued under the same external_id. Keep the + // GUID, release the assigned slot, and tombstone only listener + // redelivery of this GUID. REST EnsureAuthoritative of a still-queued + // exact identity clears that tombstone. A later different GUID with + // the same run/name is not this job and must not inherit its clock. // A fast job may start and finish in this one batch. Completion // must win; keeping its pre-start lineage would let the started // loop resurrect a terminal running intent and request spare VMs. @@ -849,88 +864,30 @@ func bindStartedIdentity(journal *queueIntentJournal, config queueAdmissionConfi } func transferTerminalWaiterLineage( - journal *queueIntentJournal, - scaleSet params.ScaleSet, - entity params.ForgeEntity, - replacement *queueIntent, - now time.Time, + _ *queueIntentJournal, + _ params.ScaleSet, + _ params.ForgeEntity, + _ *queueIntent, + _ time.Time, ) (string, bool, error) { - if replacement == nil || replacement.WorkflowRunID <= 0 || !validQueueText(replacement.JobDisplayName) { - return "", false, nil - } - candidates := make([]queueIntent, 0, 1) - for _, candidate := range journal.Intents { - terminalExpiry, terminal := journal.TerminalJobs[candidate.JobID] - if !terminal || !terminalExpiry.After(now) || candidate.JobID == replacement.JobID || - candidate.State != queueStateQueued || candidate.RunnerRequestID != 0 || candidate.RunnerName != "" || - candidate.ScaleSetID != int64(scaleSet.ScaleSetID) || candidate.ScaleSetName != scaleSet.Name || - candidate.Owner != entity.Owner || candidate.WorkflowRunID != replacement.WorkflowRunID || - candidate.JobDisplayName != replacement.JobDisplayName { - continue - } - candidates = append(candidates, candidate) - } - if len(candidates) > 1 { - return "", false, fmt.Errorf("redelivered waiter identity is ambiguous for workflow run %d job %q", replacement.WorkflowRunID, replacement.JobDisplayName) - } - if len(candidates) == 0 { - return "", false, nil - } - lineage := candidates[0] - delete(journal.Intents, lineage.Key) - replacement.QueueTime = lineage.QueueTime - replacement.FirstQueuedAt = lineage.FirstQueuedAt - replacement.Priority = lineage.Priority - replacement.Repository = lineage.Repository - replacement.State = queueStateQueued - replacement.StateEnteredAt = now - replacement.UpdatedAt = now - return lineage.JobID, true, nil + // Listener JobCompleted ends a delivery reservation. It does not prove a + // REST workflow-job terminal outcome, and a later JobAssigned with the + // same run/name is a different GUID. Do not migrate identity or queue age. + return "", false, nil } func transferAssignedReservation( - journal *queueIntentJournal, - config queueAdmissionConfig, - scaleSet params.ScaleSet, - entity params.ForgeEntity, - started params.ScaleSetJobMessage, - now time.Time, + _ *queueIntentJournal, + _ queueAdmissionConfig, + _ params.ScaleSet, + _ params.ForgeEntity, + _ params.ScaleSetJobMessage, + _ time.Time, ) (string, bool, error) { - candidates := make([]queueIntent, 0) - for _, candidate := range journal.Intents { - if candidate.State != queueStateAssigned || candidate.ScaleSetID != int64(scaleSet.ScaleSetID) || - candidate.ScaleSetName != scaleSet.Name || candidate.Owner != entity.Owner || candidate.JobID == started.JobID { - continue - } - candidates = append(candidates, candidate) - } - if len(candidates) == 0 { - return "", false, nil - } - sort.Slice(candidates, func(left, right int) bool { - if !candidates[left].UpdatedAt.Equal(candidates[right].UpdatedAt) { - return candidates[left].UpdatedAt.Before(candidates[right].UpdatedAt) - } - return candidates[left].Key < candidates[right].Key - }) - replacement, err := queueIntentFromLifecycle( - config, scaleSet, entity, started, now, time.Duration(config.ExecutionTTLSeconds)*time.Second, - ) - if err != nil { - return "", false, err - } - reservation := candidates[0] - delete(journal.Intents, reservation.Key) - replacement.QueueTime = reservation.QueueTime - replacement.FirstQueuedAt = reservation.FirstQueuedAt - replacement.Priority = reservation.Priority - replacement.State = queueStateRunning - replacement.StateEnteredAt = now - replacement.UpdatedAt = now - replacement.ExpiresAt = expiryForState(config, queueStateRunning, now) - journal.Intents[replacement.Key] = replacement - ensureRepositoryState(journal, config, replacement.Repository) - return reservation.JobID, true, nil + // JobStarted of GUID B is not proof that assigned waiter A was a + // mis-keyed reservation for B. Deleting or renaming A drops its FIFO + // clock and hides a still-waiting job. Rehydrate B on its own key. + return "", false, nil } func admitQueuedToBudget(journal *queueIntentJournal, config queueAdmissionConfig, now time.Time) { @@ -1060,6 +1017,13 @@ func (c *queueIntentCoordinator) update(config queueAdmissionConfig, mutate func if err := mutate(&journal, now); err != nil { return err } + // JobAvailable is the dispatch token. A request-less JobAssigned may occupy + // the slot to bootstrap an empty scale set, but admit above can re-grant that + // occupancy after TTL before this transaction records a newer available job. + // Yield that occupancy here so SelectForAcquire can take the dispatchable GUID + // without deleting the original waiter or its FIFO clock. + yieldRequestlessOccupancy(&journal, config, now) + admitQueuedToBudget(&journal, config, now) if err := journal.Validate(); err != nil { return err } @@ -1305,9 +1269,46 @@ func baseQueuePriority(config queueAdmissionConfig, scaleSetName string, job par return 1 } +func hasDispatchableQueued(journal *queueIntentJournal) bool { + for _, intent := range journal.Intents { + if intent.State == queueStateQueued && intent.RunnerRequestID != 0 { + return true + } + } + return false +} + +func yieldRequestlessOccupancy(journal *queueIntentJournal, config queueAdmissionConfig, now time.Time) { + if !hasDispatchableQueued(journal) { + return + } + for key, intent := range journal.Intents { + if intent.State != queueStateAssigned || intent.RunnerRequestID != 0 || intent.RunnerName != "" { + continue + } + intent.State = queueStateQueued + intent.StateEnteredAt = now + intent.UpdatedAt = now + if intent.FirstQueuedAt.IsZero() { + intent.FirstQueuedAt = intent.QueueTime + if intent.FirstQueuedAt.IsZero() { + intent.FirstQueuedAt = now + } + } + next := expiryForState(config, queueStateQueued, now) + horizon := intent.FirstQueuedAt.Add(time.Duration(config.ExecutionTTLSeconds) * time.Second) + if next.After(horizon) { + next = horizon + } + intent.ExpiresAt = next + journal.Intents[key] = intent + } +} + func eligibleQueueCandidates(journal *queueIntentJournal, config queueAdmissionConfig, inFlight map[string]int, now time.Time) []queueIntent { candidates := make([]queueIntent, 0) backgroundInFlight := queueBackgroundInFlight(journal) + dispatchableQueued := hasDispatchableQueued(journal) for _, intent := range journal.Intents { limit := repositoryPolicy(config, intent.Repository).MaxInFlight if queueHasCompetingRepository(journal, intent.Repository) { @@ -1316,7 +1317,8 @@ func eligibleQueueCandidates(journal *queueIntentJournal, config queueAdmissionC terminalExpiry, terminal := journal.TerminalJobs[intent.JobID] if intent.State != queueStateQueued || (terminal && terminalExpiry.After(now)) || inFlight[intent.Repository] >= limit || - (intent.Priority == 2 && backgroundInFlight >= config.MaxBackgroundInFlight) { + (intent.Priority == 2 && backgroundInFlight >= config.MaxBackgroundInFlight) || + (dispatchableQueued && intent.RunnerRequestID == 0) { continue } candidates = append(candidates, intent) diff --git a/third_party/garm/overlay/workers/scaleset/queue_intent_test.go b/third_party/garm/overlay/workers/scaleset/queue_intent_test.go index c940a27..a745943 100644 --- a/third_party/garm/overlay/workers/scaleset/queue_intent_test.go +++ b/third_party/garm/overlay/workers/scaleset/queue_intent_test.go @@ -185,7 +185,7 @@ func TestCompletedJobCannotBeResurrectedByDelayedAssignedRedelivery(t *testing.T } } -func TestRedeliveredWaiterInheritsOriginalQueueAge(t *testing.T) { +func TestRedeliveredWaiterDoesNotInheritCompletedQueueAge(t *testing.T) { now := time.Date(2026, 8, 31, 1, 46, 0, 0, time.UTC) coordinator := testQueueCoordinator(t, &now, nil) scaleSet := testQueueScaleSet(11, "nddev-linux-standard") @@ -193,8 +193,8 @@ func TestRedeliveredWaiterInheritsOriginalQueueAge(t *testing.T) { original.MessageType = params.MessageTypeJobAssigned original.RunnerRequestID = 0 original.RunnerName = "" - original.WorkflowRunID = 33347335156 - original.JobDisplayName = "ci-gate" + original.WorkflowRunID = 101 + original.JobDisplayName = "example-job" entity := testQueueEntityForJob(original) if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{original}, nil, nil); err != nil { t.Fatal(err) @@ -203,14 +203,17 @@ func TestRedeliveredWaiterInheritsOriginalQueueAge(t *testing.T) { if err != nil { t.Fatal(err) } - lineageQueueTime := initial.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), original.JobID)].QueueTime - if _, err := coordinator.ObserveLifecycle(scaleSet, entity, nil, nil, []params.ScaleSetJobMessage{original}); err != nil { + originalQueueTime := initial.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), original.JobID)].QueueTime + completed := original + completed.MessageType = params.MessageTypeJobCompleted + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, nil, nil, []params.ScaleSetJobMessage{completed}); err != nil { t.Fatal(err) } now = now.Add(5 * time.Minute) replacement := original replacement.JobID = "00000000-0000-4000-8000-000000000702" + replacement.MessageType = params.MessageTypeJobAssigned if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{replacement}, nil, nil); err != nil { t.Fatal(err) } @@ -218,15 +221,104 @@ func TestRedeliveredWaiterInheritsOriginalQueueAge(t *testing.T) { if err != nil { t.Fatal(err) } - if _, exists := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), original.JobID)]; exists { - t.Fatal("redelivery retained the obsolete job UUID") + originalIntent, exists := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), original.JobID)] + if !exists || originalIntent.State != queueStateQueued || !originalIntent.QueueTime.Equal(originalQueueTime) { + t.Fatalf("listener completion must keep the original GUID instead of aliasing it away: %#v", originalIntent) } intent := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), replacement.JobID)] - if intent.JobID != replacement.JobID || !intent.QueueTime.Equal(lineageQueueTime) || intent.State != queueStateAssigned { - t.Fatalf("redelivered waiter lost lineage: %#v", intent) + if intent.JobID != replacement.JobID || intent.State != queueStateAssigned { + t.Fatalf("replacement GUID was not recorded as its own assigned waiter: %#v", intent) + } + if intent.QueueTime.Equal(originalQueueTime) || intent.FirstQueuedAt.Equal(originalIntent.FirstQueuedAt) { + t.Fatalf("run/name heuristic aliased the completed waiter's queue clock: %#v", intent) } if !journal.TerminalJobs[original.JobID].After(now) { - t.Fatal("obsolete UUID lost its terminal tombstone") + t.Fatal("completed GUID lost its terminal tombstone") + } +} + +func TestListenerCompletedGUIDDoesNotBecomeRESTTerminalProof(t *testing.T) { + // Empty runner_name JobCompleted ends the scale-set delivery reservation. + // The REST workflow job may remain queued under the same external_id. + now := time.Date(2026, 9, 7, 4, 0, 0, 0, time.UTC) + coordinator := testQueueCoordinator(t, &now, nil) + scaleSet := testQueueScaleSet(11, "nddev-linux-standard") + assigned := testQueueJob(801, "example-owner", "example-repository", now) + assigned.MessageType = params.MessageTypeJobAssigned + assigned.RunnerRequestID = 0 + assigned.RunnerName = "" + entity := testQueueEntityForJob(assigned) + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{assigned}, nil, nil); err != nil { + t.Fatal(err) + } + now = now.Add(5 * time.Minute) + completed := assigned + completed.MessageType = params.MessageTypeJobCompleted + completed.RunnerName = "" + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, nil, nil, []params.ScaleSetJobMessage{completed}); err != nil { + t.Fatal(err) + } + journal, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + intent, exists := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), assigned.JobID)] + if !exists || intent.State != queueStateQueued { + t.Fatalf("listener JobCompleted is not REST terminal deletion: %#v", intent) + } + if !journal.TerminalJobs[assigned.JobID].After(now) { + t.Fatal("listener completion tombstone missing") + } + target, err := coordinator.AdmittedCapacityTarget(scaleSet, entity) + if err != nil || target != 0 { + t.Fatalf("never-started listener completion must release assigned capacity: target=%d err=%v", target, err) + } +} + +func TestAuthoritativeQueuedJobClearsListenerDeliveryTombstone(t *testing.T) { + now := time.Date(2026, 9, 7, 4, 10, 5, 0, time.UTC) + coordinator := testQueueCoordinator(t, &now, nil) + scaleSet := testQueueScaleSet(11, "nddev-linux-standard") + assigned := testQueueJob(901, "example-owner", "example-repository", now.Add(-5*time.Minute)) + assigned.MessageType = params.MessageTypeJobAssigned + assigned.RunnerRequestID = 0 + assigned.RunnerName = "" + entity := testQueueEntityForJob(assigned) + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{assigned}, nil, nil); err != nil { + t.Fatal(err) + } + completed := assigned + completed.MessageType = params.MessageTypeJobCompleted + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, nil, nil, []params.ScaleSetJobMessage{completed}); err != nil { + t.Fatal(err) + } + before, err := coordinator.AdmittedCapacityTarget(scaleSet, entity) + if err != nil || before != 0 { + t.Fatalf("listener delivery completion still occupied a slot: target=%d err=%v", before, err) + } + now = now.Add(time.Minute) + job := params.Job{ + ScaleSetJobID: assigned.JobID, + RepositoryOwner: "example-owner", RepositoryName: "example-repository", Action: "pull_request", + } + job.CreatedAt = assigned.QueueTime + if _, err := coordinator.EnsureAuthoritative(scaleSet, entity, job); err != nil { + t.Fatal(err) + } + journal, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + if _, terminal := journal.TerminalJobs[assigned.JobID]; terminal { + t.Fatal("REST still-queued identity left a listener delivery tombstone in place") + } + intent := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), assigned.JobID)] + if intent.JobID != assigned.JobID || intent.State == "" { + t.Fatalf("REST-queued GUID was not retained: %#v", intent) + } + target, err := coordinator.AdmittedCapacityTarget(scaleSet, entity) + if err != nil || target != 1 { + t.Fatalf("REST-queued exact identity must be admissible again: target=%d err=%v state=%s", target, err, intent.State) } } @@ -969,15 +1061,16 @@ func TestQueueCoordinatorTracksAStartedJobWithNoIntent(t *testing.T) { } } -func TestQueueCoordinatorTransfersAssignedCapacityToTheJobGitHubActuallyStarted(t *testing.T) { +func TestQueueCoordinatorJobStartedDoesNotConsumeUnrelatedAssignedWaiter(t *testing.T) { now := time.Date(2026, 8, 20, 20, 0, 0, 0, time.UTC) coordinator := testQueueCoordinator(t, &now, nil) scaleSet := testQueueScaleSet(11, "nddev-linux-standard") + entity := testQueueEntityForJob(testQueueJob(101, "example-org", "reserved", now)) reserved := testQueueJob(101, "example-org", "reserved", now.Add(-time.Minute)) reserved.MessageType = params.MessageTypeJobAssigned reserved.RunnerRequestID = 0 if _, err := coordinator.ObserveLifecycle( - scaleSet, testQueueEntityForJob(reserved), []params.ScaleSetJobMessage{reserved}, nil, nil, + scaleSet, entity, []params.ScaleSetJobMessage{reserved}, nil, nil, ); err != nil { t.Fatal(err) } @@ -985,10 +1078,25 @@ func TestQueueCoordinatorTransfersAssignedCapacityToTheJobGitHubActuallyStarted( if err != nil { t.Fatal(err) } - reservation := before.Intents[queueIntentKey(11, reserved.JobID)] + reservedKey := queueIntentKey(11, reserved.JobID) + reservation := before.Intents[reservedKey] + if reservation.State != queueStateAssigned { + t.Fatalf("assigned waiter was not recorded: %#v", reservation) + } started := testQueueJob(202, "example-org", "actual", now) started.MessageType = params.MessageTypeJobStarted + fromJobID, transferred, err := transferAssignedReservation( + &before, queueAdmissionConfig{}, scaleSet, entity, started, now, + ) + if err != nil || transferred || fromJobID != "" { + t.Fatalf("helper renamed a foreign waiter: transferred=%t from=%q err=%v", transferred, fromJobID, err) + } + if waiter, exists := before.Intents[reservedKey]; !exists || waiter != reservation { + t.Fatalf("helper mutated the assigned waiter: before=%#v after=%#v", reservation, waiter) + } + + now = now.Add(2 * time.Minute) if _, err := coordinator.ObserveLifecycle( scaleSet, testQueueEntityForJob(started), nil, []params.ScaleSetJobMessage{started}, nil, ); err != nil { @@ -998,15 +1106,177 @@ func TestQueueCoordinatorTransfersAssignedCapacityToTheJobGitHubActuallyStarted( if err != nil { t.Fatal(err) } - if _, exists := journal.Intents[queueIntentKey(11, reserved.JobID)]; exists { - t.Fatal("the substituted job kept a second capacity token") + waiter, exists := journal.Intents[reservedKey] + if !exists || waiter.State != queueStateAssigned || waiter.Key != reservation.Key || + waiter.QueueTime != reservation.QueueTime || waiter.FirstQueuedAt != reservation.FirstQueuedAt || + waiter.Priority != reservation.Priority || waiter.JobID != reserved.JobID { + t.Fatalf("unrelated JobStarted consumed or renamed the assigned waiter: %#v", waiter) } actual, exists := journal.Intents[queueIntentKey(11, started.JobID)] - if !exists || actual.State != queueStateRunning || actual.QueueTime != reservation.QueueTime { - t.Fatalf("capacity reservation was not transferred: %#v", actual) + if !exists || actual.State != queueStateRunning || actual.JobID != started.JobID { + t.Fatalf("started job was not rehydrated on its own key: %#v", actual) + } + if actual.QueueTime == reservation.QueueTime || actual.FirstQueuedAt == reservation.FirstQueuedAt { + t.Fatalf("started job inherited the foreign FIFO clock: started=%#v waiter=%#v", actual, waiter) + } + if total, _ := queueInFlight(&journal); total != 2 { + t.Fatalf("original waiter plus started job must both remain in-flight: %d", total) + } +} + +func TestQueueCoordinatorDispatchableAvailableTakesSlotFromRequestlessReservation(t *testing.T) { + now := time.Date(2026, 9, 7, 4, 22, 0, 0, time.UTC) + coordinator := testQueueCoordinator(t, &now, nil) + scaleSet := testQueueScaleSet(5, "nddev-linux-integration") + old := testQueueJob(101, "example-owner", "example-repository", now) + old.MessageType = params.MessageTypeJobAssigned + old.RunnerRequestID = 0 + old.RunnerName = "" + entity := testQueueEntityForJob(old) + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{old}, nil, nil); err != nil { + t.Fatal(err) + } + oldKey := queueIntentKey(int64(scaleSet.ScaleSetID), old.JobID) + before, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + reservation := before.Intents[oldKey] + if reservation.State != queueStateAssigned { + t.Fatalf("request-less assigned waiter missing: %#v", reservation) + } + + now = now.Add(11 * time.Minute) + demandNow := now + if _, err := coordinator.AdmittedCapacityTarget(scaleSet, entity); err != nil { + t.Fatal(err) + } + fresh := testQueueJob(202, "example-owner", "example-repository", now) + fresh.MessageType = params.MessageTypeJobAvailable + if err := coordinator.ObserveAvailable(scaleSet, []params.ScaleSetJobMessage{fresh}); err != nil { + t.Fatal(err) + } + selected, err := coordinator.SelectForAcquire(scaleSet, []params.ScaleSetJobMessage{fresh}) + if err != nil { + t.Fatal(err) + } + if len(selected) != 1 || selected[0] != fresh.RunnerRequestID { + t.Fatalf("dispatchable available job was not selected: %v", selected) + } + + journal, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + waiter := journal.Intents[oldKey] + if waiter.State != queueStateQueued || waiter.JobID != old.JobID || + waiter.QueueTime != reservation.QueueTime || waiter.FirstQueuedAt != reservation.FirstQueuedAt || + waiter.Priority != reservation.Priority || waiter.RunnerRequestID != 0 { + t.Fatalf("original waiter lost identity or FIFO: %#v", waiter) + } + acquired := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), fresh.JobID)] + if acquired.State != queueStateAcquiring || acquired.RunnerRequestID != fresh.RunnerRequestID { + t.Fatalf("dispatchable job did not receive occupancy: %#v", acquired) + } + if total, _ := queueInFlight(&journal); total != 1 { + t.Fatalf("request-less waiter plus dispatchable occupancy: %d", total) + } + admitted, err := coordinator.AdmittedCapacityTarget(scaleSet, entity) + if err != nil || admitted != 1 { + t.Fatalf("admitted=%d err=%v", admitted, err) + } + demand := liveMessageDemand{ + sessionID: "sess-live", assigned: 2, observed: true, + messageID: 9, observedAt: demandNow, + } + local := params.ScaleSet{Enabled: true, MaxRunners: 8, ScaleSetID: scaleSet.ScaleSetID, Name: scaleSet.Name} + target, demandErr := (&confirmedDemandGate{}).target(context.Background(), demandNow, demand, local, 0, admitted) + if demandErr != nil || target != 1 { + t.Fatalf("live demand must follow dispatchable occupancy: target=%d err=%v", target, demandErr) + } +} + +func TestQueueCoordinatorUnexpiredRequestlessYieldsToDispatchableAvailable(t *testing.T) { + now := time.Date(2026, 9, 7, 4, 22, 0, 0, time.UTC) + coordinator := testQueueCoordinator(t, &now, nil) + scaleSet := testQueueScaleSet(5, "nddev-linux-integration") + old := testQueueJob(101, "example-owner", "example-repository", now) + old.MessageType = params.MessageTypeJobAssigned + old.RunnerRequestID = 0 + old.RunnerName = "" + entity := testQueueEntityForJob(old) + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{old}, nil, nil); err != nil { + t.Fatal(err) + } + oldKey := queueIntentKey(int64(scaleSet.ScaleSetID), old.JobID) + before, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + reservation := before.Intents[oldKey] + if reservation.State != queueStateAssigned { + t.Fatalf("request-less assigned waiter missing: %#v", reservation) + } + + now = now.Add(30 * time.Second) + fresh := testQueueJob(202, "example-owner", "example-repository", now) + fresh.MessageType = params.MessageTypeJobAvailable + if err := coordinator.ObserveAvailable(scaleSet, []params.ScaleSetJobMessage{fresh}); err != nil { + t.Fatal(err) + } + selected, err := coordinator.SelectForAcquire(scaleSet, []params.ScaleSetJobMessage{fresh}) + if err != nil { + t.Fatal(err) + } + if len(selected) != 1 || selected[0] != fresh.RunnerRequestID { + t.Fatalf("unexpired request-less occupancy must yield to JobAvailable: %v", selected) + } + journal, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + waiter := journal.Intents[oldKey] + if waiter.State != queueStateQueued || waiter.QueueTime != reservation.QueueTime || + waiter.FirstQueuedAt != reservation.FirstQueuedAt || waiter.Priority != reservation.Priority { + t.Fatalf("unexpired waiter lost FIFO: %#v", waiter) + } +} + +func TestQueueCoordinatorSameGUIDJobAvailableKeepsOccupancy(t *testing.T) { + now := time.Date(2026, 9, 7, 4, 22, 0, 0, time.UTC) + coordinator := testQueueCoordinator(t, &now, nil) + scaleSet := testQueueScaleSet(5, "nddev-linux-integration") + assigned := testQueueJob(101, "example-owner", "example-repository", now) + assigned.MessageType = params.MessageTypeJobAssigned + assigned.RunnerRequestID = 0 + assigned.RunnerName = "" + entity := testQueueEntityForJob(assigned) + if _, err := coordinator.ObserveLifecycle(scaleSet, entity, []params.ScaleSetJobMessage{assigned}, nil, nil); err != nil { + t.Fatal(err) + } + available := assigned + available.MessageType = params.MessageTypeJobAvailable + available.RunnerRequestID = 909 + if err := coordinator.ObserveAvailable(scaleSet, []params.ScaleSetJobMessage{available}); err != nil { + t.Fatal(err) + } + selected, err := coordinator.SelectForAcquire(scaleSet, []params.ScaleSetJobMessage{available}) + if err != nil { + t.Fatal(err) + } + if len(selected) != 1 || selected[0] != available.RunnerRequestID { + t.Fatalf("same-GUID JobAvailable must keep occupancy: %v", selected) + } + journal, err := readQueueIntentJournal(coordinator.journalPath) + if err != nil { + t.Fatal(err) + } + intent := journal.Intents[queueIntentKey(int64(scaleSet.ScaleSetID), assigned.JobID)] + if intent.State != queueStateAcquiring || intent.RunnerRequestID != available.RunnerRequestID { + t.Fatalf("same GUID was yielded or lost its request ID: %#v", intent) } if total, _ := queueInFlight(&journal); total != 1 { - t.Fatalf("reservation transfer changed in-flight width: %d", total) + t.Fatalf("same GUID must occupy exactly one slot: %d", total) } } diff --git a/third_party/garm/patches/0032-exact-scale-set-job-identity.patch b/third_party/garm/patches/0032-exact-scale-set-job-identity.patch new file mode 100644 index 0000000..864ba3f --- /dev/null +++ b/third_party/garm/patches/0032-exact-scale-set-job-identity.patch @@ -0,0 +1,1319 @@ +diff --git a/runner/common/mocks/GithubClient.go b/runner/common/mocks/GithubClient.go +index 5b8f3d4..c2f11fd 100644 +--- a/runner/common/mocks/GithubClient.go ++++ b/runner/common/mocks/GithubClient.go +@@ -582,6 +582,77 @@ func (_c *GithubClient_GetWorkflowRunByID_Call) RunAndReturn(run func(context.Co + return _c + } + ++// ListCheckRunsCheckSuite provides a mock function with given fields: ctx, owner, repo, checkSuiteID, opts ++func (_m *GithubClient) ListCheckRunsCheckSuite(ctx context.Context, owner string, repo string, checkSuiteID int64, opts *github.ListCheckRunsOptions) (*github.ListCheckRunsResults, *github.Response, error) { ++ ret := _m.Called(ctx, owner, repo, checkSuiteID, opts) ++ ++ if len(ret) == 0 { ++ panic("no return value specified for ListCheckRunsCheckSuite") ++ } ++ ++ var r0 *github.ListCheckRunsResults ++ var r1 *github.Response ++ var r2 error ++ if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, *github.ListCheckRunsOptions) (*github.ListCheckRunsResults, *github.Response, error)); ok { ++ return rf(ctx, owner, repo, checkSuiteID, opts) ++ } ++ if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, *github.ListCheckRunsOptions) *github.ListCheckRunsResults); ok { ++ r0 = rf(ctx, owner, repo, checkSuiteID, opts) ++ } else { ++ if ret.Get(0) != nil { ++ r0 = ret.Get(0).(*github.ListCheckRunsResults) ++ } ++ } ++ ++ if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, *github.ListCheckRunsOptions) *github.Response); ok { ++ r1 = rf(ctx, owner, repo, checkSuiteID, opts) ++ } else { ++ if ret.Get(1) != nil { ++ r1 = ret.Get(1).(*github.Response) ++ } ++ } ++ ++ if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, *github.ListCheckRunsOptions) error); ok { ++ r2 = rf(ctx, owner, repo, checkSuiteID, opts) ++ } else { ++ r2 = ret.Error(2) ++ } ++ ++ return r0, r1, r2 ++} ++ ++// GithubClient_ListCheckRunsCheckSuite_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListCheckRunsCheckSuite' ++type GithubClient_ListCheckRunsCheckSuite_Call struct { ++ *mock.Call ++} ++ ++// ListCheckRunsCheckSuite is a helper method to define mock.On call ++// - ctx context.Context ++// - owner string ++// - repo string ++// - checkSuiteID int64 ++// - opts *github.ListCheckRunsOptions ++func (_e *GithubClient_Expecter) ListCheckRunsCheckSuite(ctx interface{}, owner interface{}, repo interface{}, checkSuiteID interface{}, opts interface{}) *GithubClient_ListCheckRunsCheckSuite_Call { ++ return &GithubClient_ListCheckRunsCheckSuite_Call{Call: _e.mock.On("ListCheckRunsCheckSuite", ctx, owner, repo, checkSuiteID, opts)} ++} ++ ++func (_c *GithubClient_ListCheckRunsCheckSuite_Call) Run(run func(ctx context.Context, owner string, repo string, checkSuiteID int64, opts *github.ListCheckRunsOptions)) *GithubClient_ListCheckRunsCheckSuite_Call { ++ _c.Call.Run(func(args mock.Arguments) { ++ run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(int64), args[4].(*github.ListCheckRunsOptions)) ++ }) ++ return _c ++} ++ ++func (_c *GithubClient_ListCheckRunsCheckSuite_Call) Return(_a0 *github.ListCheckRunsResults, _a1 *github.Response, _a2 error) *GithubClient_ListCheckRunsCheckSuite_Call { ++ _c.Call.Return(_a0, _a1, _a2) ++ return _c ++} ++ ++func (_c *GithubClient_ListCheckRunsCheckSuite_Call) RunAndReturn(run func(context.Context, string, string, int64, *github.ListCheckRunsOptions) (*github.ListCheckRunsResults, *github.Response, error)) *GithubClient_ListCheckRunsCheckSuite_Call { ++ _c.Call.Return(run) ++ return _c ++} ++ + // GithubBaseURL provides a mock function with no fields + func (_m *GithubClient) GithubBaseURL() *url.URL { + ret := _m.Called() +@@ -903,6 +974,51 @@ func (_c *GithubClient_ListWorkflowJobs_Call) RunAndReturn(run func(context.Cont + return _c + } + ++// ListWorkflowJobsAttempt provides a mock function with given fields: ctx, owner, repo, runID, attemptNumber, opts ++func (_m *GithubClient) ListWorkflowJobsAttempt(ctx context.Context, owner string, repo string, runID int64, attemptNumber int64, opts *github.ListOptions) (*github.Jobs, *github.Response, error) { ++ ret := _m.Called(ctx, owner, repo, runID, attemptNumber, opts) ++ ++ if len(ret) == 0 { ++ panic("no return value specified for ListWorkflowJobsAttempt") ++ } ++ ++ var r0 *github.Jobs ++ var r1 *github.Response ++ var r2 error ++ if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64, *github.ListOptions) (*github.Jobs, *github.Response, error)); ok { ++ return rf(ctx, owner, repo, runID, attemptNumber, opts) ++ } ++ if rf, ok := ret.Get(0).(func(context.Context, string, string, int64, int64, *github.ListOptions) *github.Jobs); ok { ++ r0 = rf(ctx, owner, repo, runID, attemptNumber, opts) ++ } else if ret.Get(0) != nil { ++ r0 = ret.Get(0).(*github.Jobs) ++ } ++ if rf, ok := ret.Get(1).(func(context.Context, string, string, int64, int64, *github.ListOptions) *github.Response); ok { ++ r1 = rf(ctx, owner, repo, runID, attemptNumber, opts) ++ } else if ret.Get(1) != nil { ++ r1 = ret.Get(1).(*github.Response) ++ } ++ if rf, ok := ret.Get(2).(func(context.Context, string, string, int64, int64, *github.ListOptions) error); ok { ++ r2 = rf(ctx, owner, repo, runID, attemptNumber, opts) ++ } else { ++ r2 = ret.Error(2) ++ } ++ return r0, r1, r2 ++} ++ ++type GithubClient_ListWorkflowJobsAttempt_Call struct { ++ *mock.Call ++} ++ ++func (_e *GithubClient_Expecter) ListWorkflowJobsAttempt(ctx interface{}, owner interface{}, repo interface{}, runID interface{}, attemptNumber interface{}, opts interface{}) *GithubClient_ListWorkflowJobsAttempt_Call { ++ return &GithubClient_ListWorkflowJobsAttempt_Call{Call: _e.mock.On("ListWorkflowJobsAttempt", ctx, owner, repo, runID, attemptNumber, opts)} ++} ++ ++func (_c *GithubClient_ListWorkflowJobsAttempt_Call) Return(_a0 *github.Jobs, _a1 *github.Response, _a2 error) *GithubClient_ListWorkflowJobsAttempt_Call { ++ _c.Call.Return(_a0, _a1, _a2) ++ return _c ++} ++ + // PingEntityHook provides a mock function with given fields: ctx, id + func (_m *GithubClient) PingEntityHook(ctx context.Context, id int64) (*github.Response, error) { + ret := _m.Called(ctx, id) +diff --git a/runner/common/util.go b/runner/common/util.go +index 2ad20a3..eccfc6f 100644 +--- a/runner/common/util.go ++++ b/runner/common/util.go +@@ -58,4 +58,6 @@ type GithubClient interface { + GetWorkflowJobByID(ctx context.Context, owner, repo string, jobID int64) (*github.WorkflowJob, *github.Response, error) + GetWorkflowRunByID(ctx context.Context, owner, repo string, runID int64) (*github.WorkflowRun, *github.Response, error) + ListWorkflowJobs(ctx context.Context, owner, repo string, runID int64, opts *github.ListWorkflowJobsOptions) (*github.Jobs, *github.Response, error) ++ ListWorkflowJobsAttempt(ctx context.Context, owner, repo string, runID, attemptNumber int64, opts *github.ListOptions) (*github.Jobs, *github.Response, error) ++ ListCheckRunsCheckSuite(ctx context.Context, owner, repo string, checkSuiteID int64, opts *github.ListCheckRunsOptions) (*github.ListCheckRunsResults, *github.Response, error) + } +diff --git a/runner/pool/pool.go b/runner/pool/pool.go +index 7d8a5ce..7f2635a 100644 +--- a/runner/pool/pool.go ++++ b/runner/pool/pool.go +@@ -2139,98 +2139,235 @@ func (r *basePoolManager) reconcileStaleJobs() error { + return nil + } + +-// reconcileStaleScaleSetJob returns true only when GitHub authoritatively proves +-// that a queued scale-set job is absent or no longer queued. Ambiguous matrix +-// jobs are retained so reconciliation cannot delete live work. ++// reconcileStaleScaleSetJob mutates a queued scale-set DB row only after the ++// check-run external GUID, repository, run, exact attempt, check_run_url and ++// source SHA bind to one GitHub Actions workflow job. A name is never identity. + func (r *basePoolManager) reconcileStaleScaleSetJob(job params.Job) bool { +- if job.ScaleSetJobID == "" || job.RunID == 0 || job.Name == "" { +- slog.WarnContext(r.ctx, "scale set job is missing reconciliation identity", "scale_set_job_id", job.ScaleSetJobID) ++ if job.ScaleSetJobID == "" || job.RunID == 0 || job.RepositoryOwner == "" || job.RepositoryName == "" { ++ slog.WarnContext(r.ctx, "scale set job is missing exact reconciliation identity", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ forgeType, err := r.entity.GetForgeType() ++ if err != nil || forgeType != params.GithubEndpointType { ++ slog.WarnContext(r.ctx, "scale set job identity is unsupported on this provider", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if r.entity.EntityType == params.ForgeEntityTypeRepository && ++ (!strings.EqualFold(r.entity.Owner, job.RepositoryOwner) || !strings.EqualFold(r.entity.Name, job.RepositoryName)) { ++ slog.WarnContext(r.ctx, "scale set job repository does not match pool entity", "scale_set_job_id", job.ScaleSetJobID) + return false + } + +- matched := make([]*github.WorkflowJob, 0, 1) +- opts := &github.ListWorkflowJobsOptions{Filter: "latest", ListOptions: github.ListOptions{PerPage: 100}} +- for { +- jobs, resp, err := r.ghcli.ListWorkflowJobs(r.ctx, job.RepositoryOwner, job.RepositoryName, job.RunID, opts) +- if err != nil { +- if resp != nil && resp.StatusCode == http.StatusNotFound { +- return r.deleteStaleScaleSetJob(job, "workflow run no longer exists", true) +- } +- if resp != nil && r.deferAuthoritativeReconcile(time.Now(), resp.StatusCode) { +- slog.WarnContext(r.ctx, "authoritative job reconciliation deferred after GitHub access refusal", +- "status_code", resp.StatusCode, +- "backoff", authoritativeReconcileAccessBackoff) +- } +- slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to list workflow run jobs", "run_id", job.RunID) +- return false +- } +- for _, candidate := range jobs.Jobs { +- if candidate.GetName() == job.Name { +- matched = append(matched, candidate) +- } +- } +- if resp == nil || resp.NextPage == 0 { +- break +- } +- opts.Page = resp.NextPage ++ workflowRun, resp, err := r.ghcli.GetWorkflowRunByID(r.ctx, job.RepositoryOwner, job.RepositoryName, job.RunID) ++ if err != nil { ++ r.noteAuthoritativeScaleSetRead(resp, err, "failed to get workflow run", job) ++ return false ++ } ++ if workflowRun == nil || workflowRun.GetID() != job.RunID || workflowRun.GetCheckSuiteID() == 0 || ++ workflowRun.GetHeadSHA() == "" || workflowRun.GetRunAttempt() <= 0 { ++ slog.WarnContext(r.ctx, "workflow run is missing exact identity fields", "scale_set_job_id", job.ScaleSetJobID, "run_id", job.RunID) ++ return false ++ } ++ repo := workflowRun.GetRepository() ++ if repo == nil || repo.GetOwner() == nil || repo.GetOwner().GetLogin() == "" || repo.GetName() == "" { ++ slog.WarnContext(r.ctx, "workflow run is missing repository identity", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if !strings.EqualFold(repo.GetOwner().GetLogin(), job.RepositoryOwner) || !strings.EqualFold(repo.GetName(), job.RepositoryName) { ++ slog.WarnContext(r.ctx, "workflow run repository does not match scale set job", "scale_set_job_id", job.ScaleSetJobID) ++ return false + } + +- if len(matched) == 0 { +- return r.deleteStaleScaleSetJob(job, "job is absent from workflow run", true) ++ checkRun, complete, err := r.listExactScaleSetCheckRun(job, workflowRun) ++ if err != nil || !complete { ++ slog.WarnContext(r.ctx, "scale set job identity is incomplete or unknown; retaining intent", ++ "scale_set_job_id", job.ScaleSetJobID, "run_id", job.RunID) ++ return false + } +- if len(matched) > 1 { +- slog.WarnContext(r.ctx, "ambiguous scale set job retained", "scale_set_job_id", job.ScaleSetJobID, "matches", len(matched)) ++ if !isGitHubActionsCheckRun(checkRun) { ++ slog.WarnContext(r.ctx, "matched check run is not a GitHub Actions producer", "scale_set_job_id", job.ScaleSetJobID) + return false + } +- if matched[0].GetStatus() != "queued" { +- if matched[0].GetStatus() == "in_progress" { +- if err := r.ensureAuthoritativeQueueIntent(job); err != nil { +- slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to bind authoritative running job", "scale_set_job_id", job.ScaleSetJobID) +- return false +- } +- return r.deleteStaleScaleSetJob(job, "job is no longer queued", false) +- } +- return r.deleteStaleScaleSetJob(job, "job is no longer queued", true) ++ if checkRun.GetHeadSHA() == "" || checkRun.GetID() == 0 { ++ slog.WarnContext(r.ctx, "matched check run is missing identity fields", "scale_set_job_id", job.ScaleSetJobID) ++ return false + } +- workflowRun, resp, err := r.ghcli.GetWorkflowRunByID(r.ctx, job.RepositoryOwner, job.RepositoryName, job.RunID) ++ if checkRun.CheckSuite == nil || checkRun.CheckSuite.GetID() != workflowRun.GetCheckSuiteID() { ++ slog.WarnContext(r.ctx, "check run suite does not match workflow run", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ ++ listedJob, complete, err := r.listExactAttemptJobForCheckRun(job, workflowRun, checkRun) ++ if err != nil || !complete { ++ slog.WarnContext(r.ctx, "exact-attempt workflow job is incomplete or unknown; retaining intent", ++ "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ ghJob, resp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.RepositoryOwner, job.RepositoryName, listedJob.GetID()) + if err != nil { +- if resp != nil && resp.StatusCode == http.StatusNotFound { +- return r.deleteStaleScaleSetJob(job, "workflow run no longer exists", true) ++ r.noteAuthoritativeScaleSetRead(resp, err, "failed to get workflow job bound by check_run_url", job) ++ return false ++ } ++ if ghJob == nil || ghJob.GetID() != listedJob.GetID() || ghJob.GetID() == 0 || ghJob.GetRunID() != job.RunID { ++ slog.WarnContext(r.ctx, "workflow job does not bind to the matched check run URL", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if !workflowJobBindsCheckRun(ghJob, checkRun) { ++ slog.WarnContext(r.ctx, "workflow job check_run_url does not match the check run", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if ghJob.GetRunAttempt() <= 0 || ghJob.GetRunAttempt() != int64(workflowRun.GetRunAttempt()) { ++ slog.WarnContext(r.ctx, "workflow job attempt does not match workflow run", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if _, ok := exactHeadSHA(workflowRun.GetHeadSHA(), checkRun.GetHeadSHA(), ghJob.GetHeadSHA()); !ok { ++ slog.WarnContext(r.ctx, "scale set job source SHA is missing or mismatched", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if len(ghJob.Labels) == 0 { ++ slog.WarnContext(r.ctx, "matched job has no scale-set labels", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ if _, err := r.exactScaleSetForJobFromLabels(ghJob.Labels); err != nil { ++ slog.With(slog.Any("error", err)).WarnContext(r.ctx, "matched job labels do not uniquely identify an entity scale set", "scale_set_job_id", job.ScaleSetJobID) ++ return false ++ } ++ ++ switch ghJob.GetStatus() { ++ case "queued": ++ if err := r.ensureAuthoritativeQueueIntent(job); err != nil { ++ slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to rehydrate authoritative queued job", "scale_set_job_id", job.ScaleSetJobID) ++ } ++ return false ++ case "in_progress": ++ if err := r.ensureAuthoritativeQueueIntent(job); err != nil { ++ slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to bind authoritative running job", "scale_set_job_id", job.ScaleSetJobID) ++ return false + } +- if resp != nil && r.deferAuthoritativeReconcile(time.Now(), resp.StatusCode) { +- slog.WarnContext(r.ctx, "authoritative workflow run reconciliation deferred after GitHub access refusal", +- "status_code", resp.StatusCode, +- "backoff", authoritativeReconcileAccessBackoff) ++ return r.deleteStaleScaleSetJob(job, "exact scale set job is in progress", false) ++ case "completed": ++ if !validTerminalConclusion(ghJob.GetConclusion()) { ++ slog.WarnContext(r.ctx, "completed job is missing a valid terminal conclusion; retaining intent", ++ "scale_set_job_id", job.ScaleSetJobID, "conclusion", ghJob.GetConclusion()) ++ return false + } +- slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to get workflow run", "run_id", job.RunID) ++ return r.deleteStaleScaleSetJob(job, "exact scale set job is completed", true) ++ default: ++ slog.WarnContext(r.ctx, "exact scale set job has unknown status; retaining intent", ++ "scale_set_job_id", job.ScaleSetJobID, "github_status", ghJob.GetStatus()) + return false + } +- if workflowRun.GetStatus() == "completed" { +- return r.deleteStaleScaleSetJob(job, "workflow run is completed", true) ++} ++ ++func (r *basePoolManager) noteAuthoritativeScaleSetRead(resp *github.Response, err error, message string, job params.Job) { ++ if resp != nil && r.deferAuthoritativeReconcile(time.Now(), resp.StatusCode) { ++ slog.WarnContext(r.ctx, "authoritative scale set job reconciliation deferred after GitHub access refusal", ++ "status_code", resp.StatusCode, ++ "backoff", authoritativeReconcileAccessBackoff, ++ "scale_set_job_id", job.ScaleSetJobID) + } +- if err := r.ensureAuthoritativeQueueIntent(job); err != nil { +- slog.With(slog.Any("error", err)).ErrorContext(r.ctx, "failed to rehydrate authoritative queued job", "scale_set_job_id", job.ScaleSetJobID) ++ slog.With(slog.Any("error", err)).ErrorContext(r.ctx, message, "run_id", job.RunID, "scale_set_job_id", job.ScaleSetJobID) ++} ++ ++func (r *basePoolManager) listExactScaleSetCheckRun(job params.Job, workflowRun *github.WorkflowRun) (*github.CheckRun, bool, error) { ++ opts := &github.ListCheckRunsOptions{ ++ Filter: github.Ptr("all"), ++ ListOptions: github.ListOptions{ ++ PerPage: scaleSetListPageSize, ++ }, + } +- return false ++ cursor := newGitHubListCursor() ++ var matched *github.CheckRun ++ requestedPage := 0 ++ for page := 1; page <= scaleSetListMaxPages; page++ { ++ results, resp, err := r.ghcli.ListCheckRunsCheckSuite(r.ctx, job.RepositoryOwner, job.RepositoryName, workflowRun.GetCheckSuiteID(), opts) ++ if err != nil { ++ r.noteAuthoritativeScaleSetRead(resp, err, "failed to list check runs for workflow suite", job) ++ return nil, false, err ++ } ++ if results == nil { ++ return nil, false, nil ++ } ++ ids := make([]int64, 0, len(results.CheckRuns)) ++ for _, candidate := range results.CheckRuns { ++ if candidate == nil { ++ return nil, false, nil ++ } ++ ids = append(ids, candidate.GetID()) ++ if candidate.GetExternalID() != job.ScaleSetJobID { ++ continue ++ } ++ if matched != nil && matched.GetID() != candidate.GetID() { ++ return nil, false, nil ++ } ++ matched = candidate ++ } ++ done, ok := cursor.accept(requestedPage, resp, ids, results.Total) ++ if !ok { ++ return nil, false, nil ++ } ++ if done { ++ if matched == nil { ++ return nil, false, nil ++ } ++ return matched, true, nil ++ } ++ requestedPage = resp.NextPage ++ opts.Page = resp.NextPage ++ } ++ return nil, false, nil + } + +-func exactScaleSetForJob(scaleSets []params.ScaleSet, job params.Job) (params.ScaleSet, error) { +- matches := make([]params.ScaleSet, 0, 1) +- for _, scaleSet := range scaleSets { +- if !scaleSet.Enabled { +- continue ++func (r *basePoolManager) listExactAttemptJobForCheckRun(job params.Job, workflowRun *github.WorkflowRun, checkRun *github.CheckRun) (*github.WorkflowJob, bool, error) { ++ opts := &github.ListOptions{PerPage: scaleSetListPageSize} ++ cursor := newGitHubListCursor() ++ var matched *github.WorkflowJob ++ requestedPage := 0 ++ for page := 1; page <= scaleSetListMaxPages; page++ { ++ results, resp, err := r.ghcli.ListWorkflowJobsAttempt(r.ctx, job.RepositoryOwner, job.RepositoryName, job.RunID, int64(workflowRun.GetRunAttempt()), opts) ++ if err != nil { ++ r.noteAuthoritativeScaleSetRead(resp, err, "failed to list jobs for workflow run attempt", job) ++ return nil, false, err + } +- for _, label := range job.Labels { +- if label == scaleSet.Name { +- matches = append(matches, scaleSet) +- break ++ if results == nil { ++ return nil, false, nil ++ } ++ ids := make([]int64, 0, len(results.Jobs)) ++ for _, candidate := range results.Jobs { ++ if candidate == nil { ++ return nil, false, nil + } ++ ids = append(ids, candidate.GetID()) ++ if !workflowJobBindsCheckRun(candidate, checkRun) { ++ continue ++ } ++ if matched != nil && matched.GetID() != candidate.GetID() { ++ return nil, false, nil ++ } ++ matched = candidate ++ } ++ done, ok := cursor.accept(requestedPage, resp, ids, results.TotalCount) ++ if !ok { ++ return nil, false, nil ++ } ++ if done { ++ if matched == nil { ++ return nil, false, nil ++ } ++ return matched, true, nil + } ++ requestedPage = resp.NextPage ++ opts.Page = resp.NextPage + } +- if len(matches) != 1 { +- return params.ScaleSet{}, fmt.Errorf("queued job matches %d enabled scale sets", len(matches)) ++ return nil, false, nil ++} ++ ++func (r *basePoolManager) exactScaleSetForJobFromLabels(labels []string) (params.ScaleSet, error) { ++ scaleSets, err := r.store.ListEntityScaleSets(r.ctx, r.entity) ++ if err != nil { ++ return params.ScaleSet{}, fmt.Errorf("list entity scale sets: %w", err) + } +- return matches[0], nil ++ return exactScaleSetForJob(scaleSets, params.Job{Labels: labels}) + } + + func (r *basePoolManager) ensureAuthoritativeQueueIntent(job params.Job) error { +diff --git a/runner/pool/pool_test.go b/runner/pool/pool_test.go +index 53ee134..362c65c 100644 +--- a/runner/pool/pool_test.go ++++ b/runner/pool/pool_test.go +@@ -18,8 +18,11 @@ package pool + + import ( + "context" ++ "errors" + "fmt" ++ "net/http" + "sync" ++ "sync/atomic" + "testing" + "time" + +@@ -833,77 +836,215 @@ func (s *PoolStressTestSuite) TestExactScaleSetForJobFailsClosedOnMissingOrAmbig + job.Labels = []string{"nddev-linux-standard", "nddev-linux-integration"} + _, err = exactScaleSetForJob(sets, job) + s.Error(err) ++ job.Labels = nil ++ _, err = exactScaleSetForJob(sets, job) ++ s.Error(err) + } + +-func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobPaginatesAndDeletesAbsentJob() { +- reconciledJobID := "" +- nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, jobID string) (bool, error) { +- reconciledJobID = jobID +- return true, nil ++func scaleSetIdentityRun(runID, suiteID int64, sha string, attempt int) *github.WorkflowRun { ++ return &github.WorkflowRun{ ++ ID: github.Ptr(runID), ++ CheckSuiteID: github.Ptr(suiteID), ++ HeadSHA: github.Ptr(sha), ++ RunAttempt: github.Ptr(attempt), ++ Status: github.Ptr("in_progress"), ++ Repository: &github.Repository{ ++ Name: github.Ptr("test-repo"), ++ Owner: &github.User{Login: github.Ptr("test-owner")}, ++ }, ++ } ++} ++ ++func scaleSetIdentityCheckRun(id int64, externalID, status, sha string, suiteID int64) *github.CheckRun { ++ return &github.CheckRun{ ++ ID: github.Ptr(id), ++ ExternalID: github.Ptr(externalID), ++ Status: github.Ptr(status), ++ HeadSHA: github.Ptr(sha), ++ URL: github.Ptr(fmt.Sprintf("https://api.github.com/repos/test-owner/test-repo/check-runs/%d", id)), ++ CheckSuite: &github.CheckSuite{ID: github.Ptr(suiteID)}, ++ Name: github.Ptr("wanted"), ++ App: &github.App{Slug: github.Ptr("github-actions")}, ++ } ++} ++ ++func scaleSetIdentityJob(id, checkID, runID, attempt int64, status, conclusion, sha string, labels []string) *github.WorkflowJob { ++ return &github.WorkflowJob{ ++ ID: github.Ptr(id), ++ RunID: github.Ptr(runID), ++ RunAttempt: github.Ptr(attempt), ++ Status: github.Ptr(status), ++ Conclusion: github.Ptr(conclusion), ++ HeadSHA: github.Ptr(sha), ++ Name: github.Ptr("wanted"), ++ Labels: labels, ++ CheckRunURL: github.Ptr(fmt.Sprintf("https://api.github.com/repos/test-owner/test-repo/check-runs/%d", checkID)), + } ++} ++ ++func completeCheckPage(runs []*github.CheckRun, total, next int) (*github.ListCheckRunsResults, *github.Response) { ++ return &github.ListCheckRunsResults{Total: github.Ptr(total), CheckRuns: runs}, &github.Response{NextPage: next} ++} ++ ++func completeJobPage(jobs []*github.WorkflowJob, total, next int) (*github.Jobs, *github.Response) { ++ return &github.Jobs{TotalCount: github.Ptr(total), Jobs: jobs}, &github.Response{NextPage: next} ++} ++ ++func allCheckRuns(opts *github.ListCheckRunsOptions) bool { ++ return opts != nil && opts.Filter != nil && *opts.Filter == "all" && opts.CheckName == nil && opts.PerPage == 100 ++} ++ ++func (s *PoolStressTestSuite) storeQueuedScaleSetJob(id string, runID int64, name string) params.Job { ++ s.T().Helper() + job := params.Job{ +- ScaleSetJobID: "scale-job-1", RunID: 4242, Name: "wanted", ++ ScaleSetJobID: id, RunID: runID, Name: name, + Status: string(params.JobStatusQueued), RepositoryOwner: "test-owner", RepositoryName: "test-repo", ++ Labels: []string{"nddev-linux-standard"}, + } + _, err := s.store.CreateOrUpdateJob(s.adminCtx, job) + s.Require().NoError(err) ++ return job ++} + +- s.ghcliMock.EXPECT().ListWorkflowJobs(mock.Anything, "test-owner", "test-repo", int64(4242), mock.MatchedBy(func(opts *github.ListWorkflowJobsOptions) bool { +- return opts.Page == 0 && opts.PerPage == 100 && opts.Filter == "latest" +- })).Return(&github.Jobs{Jobs: []*github.WorkflowJob{{Name: github.Ptr("other")}}}, &github.Response{NextPage: 2}, nil).Once() +- s.ghcliMock.EXPECT().ListWorkflowJobs(mock.Anything, "test-owner", "test-repo", int64(4242), mock.MatchedBy(func(opts *github.ListWorkflowJobsOptions) bool { +- return opts.Page == 2 +- })).Return(&github.Jobs{}, &github.Response{}, nil).Once() +- +- s.True(s.mgr.reconcileStaleScaleSetJob(job)) +- s.Equal(job.ScaleSetJobID, reconciledJobID) +- jobs, err := s.store.ListAllJobs(s.adminCtx) ++func (s *PoolStressTestSuite) requireScaleSet() { ++ s.T().Helper() ++ _, err := s.store.CreateEntityScaleSet(s.adminCtx, s.entity, params.CreateScaleSetParams{ ++ ProviderName: "test-provider", Name: "nddev-linux-standard", Image: "test-image", Flavor: "test-flavor", ++ MaxRunners: 1, OSType: commonParams.Linux, OSArch: commonParams.Amd64, Enabled: true, ++ }) + s.Require().NoError(err) +- s.Empty(jobs) + } + +-func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsAmbiguousMatrixJob() { +- job := params.Job{ +- ScaleSetJobID: "scale-job-2", RunID: 4343, Name: "matrix", +- Status: string(params.JobStatusQueued), RepositoryOwner: "test-owner", RepositoryName: "test-repo", ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsUnknownGUIDAfterCompleteSuite() { ++ released := false ++ nddevRemoveAuthoritativeQueueIntent = func(context.Context, string) (bool, error) { ++ released = true ++ return true, nil + } +- s.ghcliMock.EXPECT().ListWorkflowJobs(mock.Anything, "test-owner", "test-repo", int64(4343), mock.Anything). +- Return(&github.Jobs{Jobs: []*github.WorkflowJob{ +- {Name: github.Ptr("matrix"), Status: github.Ptr("completed")}, +- {Name: github.Ptr("matrix"), Status: github.Ptr("queued")}, +- }}, &github.Response{}, nil).Once() ++ job := s.storeQueuedScaleSetJob("scale-job-unknown", 4242, "wanted") ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4242)). ++ Return(scaleSetIdentityRun(4242, 9001, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1), &github.Response{}, nil).Once() ++ page1, resp1 := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(101, "replacement-same-name", "completed", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 9001), ++ }, 2, 2) ++ page2, resp2 := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(102, "unrelated", "queued", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 9001), ++ }, 2, 0) ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9001), mock.MatchedBy(func(opts *github.ListCheckRunsOptions) bool { ++ return allCheckRuns(opts) && opts.Page == 0 ++ })).Return(page1, resp1, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9001), mock.MatchedBy(func(opts *github.ListCheckRunsOptions) bool { ++ return allCheckRuns(opts) && opts.Page == 2 ++ })).Return(page2, resp2, nil).Once() + + s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ s.False(released) ++ jobs, err := s.store.ListAllJobs(s.adminCtx) ++ s.Require().NoError(err) ++ s.Require().Len(jobs, 1) ++ s.Equal(job.ScaleSetJobID, jobs[0].ScaleSetJobID) + } + +-func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobDeletesQueuedChildOfCompletedRun() { ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsWrongAttemptSameName() { + released := false +- nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, jobID string) (bool, error) { +- released = jobID == "scale-job-terminal-run" ++ nddevRemoveAuthoritativeQueueIntent = func(context.Context, string) (bool, error) { ++ released = true + return true, nil + } +- job := params.Job{ +- ScaleSetJobID: "scale-job-terminal-run", RunID: 4545, Name: "cargo-audit", +- Status: string(params.JobStatusQueued), RepositoryOwner: "test-owner", RepositoryName: "test-repo", ++ job := s.storeQueuedScaleSetJob("scale-job-attempt-1", 4343, "matrix") ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4343)). ++ Return(scaleSetIdentityRun(4343, 9002, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 2), &github.Response{}, nil).Once() ++ page, resp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(201, "other-attempt-guid", "completed", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 9002), ++ }, 1, 0) ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9002), mock.MatchedBy(allCheckRuns)). ++ Return(page, resp, nil).Once() ++ ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ s.False(released) ++ jobs, err := s.store.ListAllJobs(s.adminCtx) ++ s.Require().NoError(err) ++ s.Require().Len(jobs, 1) ++ s.Equal("scale-job-attempt-1", jobs[0].ScaleSetJobID) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsWrongExternalID() { ++ job := s.storeQueuedScaleSetJob("scale-job-expected", 4440, "wanted") ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4440)). ++ Return(scaleSetIdentityRun(4440, 9003, "cccccccccccccccccccccccccccccccccccccccc", 1), &github.Response{}, nil).Once() ++ page, resp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(301, "scale-job-other", "queued", "cccccccccccccccccccccccccccccccccccccccc", 9003), ++ }, 1, 0) ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9003), mock.MatchedBy(allCheckRuns)). ++ Return(page, resp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ jobs, err := s.store.ListAllJobs(s.adminCtx) ++ s.Require().NoError(err) ++ s.Len(jobs, 1) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsMissingFieldsAndAccessRefusal() { ++ job := params.Job{ScaleSetJobID: "scale-job-incomplete", RunID: 0, RepositoryOwner: "test-owner", RepositoryName: "test-repo"} ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ ++ job = s.storeQueuedScaleSetJob("scale-job-no-name-ok", 4500, "") ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4500)). ++ Return(&github.WorkflowRun{ID: github.Ptr(int64(4500)), Status: github.Ptr("in_progress")}, &github.Response{}, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ ++ job = s.storeQueuedScaleSetJob("scale-job-forbidden", 4501, "wanted") ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4501)). ++ Return((*github.WorkflowRun)(nil), &github.Response{Response: &http.Response{StatusCode: http.StatusForbidden}}, errors.New("forbidden")).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsUnsupportedProvider() { ++ job := s.storeQueuedScaleSetJob("scale-job-gitea", 4600, "wanted") ++ original := s.mgr.entity.Credentials.ForgeType ++ s.mgr.entity.Credentials.ForgeType = params.GiteaEndpointType ++ s.T().Cleanup(func() { s.mgr.entity.Credentials.ForgeType = original }) ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobKeepsQueuedExactMatch() { ++ s.requireScaleSet() ++ hydrated := false ++ nddevEnsureAuthoritativeQueueIntent = func(_ context.Context, _ params.ScaleSet, _ params.ForgeEntity, got params.Job) (bool, error) { ++ hydrated = got.ScaleSetJobID == "scale-job-queued" ++ return true, nil + } +- _, err := s.store.CreateOrUpdateJob(s.adminCtx, job) ++ job := s.storeQueuedScaleSetJob("scale-job-queued", 4242, "") ++ sha := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ++ s.expectBoundScaleSetIdentity(4242, 9001, 900, 800, 1, sha, "scale-job-queued", "queued", "") ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++ s.True(hydrated) ++ jobs, err := s.store.ListAllJobs(s.adminCtx) + s.Require().NoError(err) +- s.ghcliMock.EXPECT().ListWorkflowJobs(mock.Anything, "test-owner", "test-repo", int64(4545), mock.Anything). +- Return(&github.Jobs{Jobs: []*github.WorkflowJob{{Name: github.Ptr("cargo-audit"), Status: github.Ptr("queued")}}}, &github.Response{}, nil).Once() +- s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4545)). +- Return(&github.WorkflowRun{Status: github.Ptr("completed"), Conclusion: github.Ptr("cancelled")}, &github.Response{}, nil).Once() ++ s.Len(jobs, 1) ++} + ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobDeletesExactCompletedMatch() { ++ s.requireScaleSet() ++ released := "" ++ nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, jobID string) (bool, error) { ++ released = jobID ++ return true, nil ++ } ++ job := s.storeQueuedScaleSetJob("scale-job-terminal", 4545, "cargo-audit") ++ sha := "dddddddddddddddddddddddddddddddddddddddd" ++ s.expectBoundScaleSetIdentity(4545, 9010, 501, 8001, 1, sha, "scale-job-terminal", "completed", "cancelled") + s.True(s.mgr.reconcileStaleScaleSetJob(job)) +- s.True(released) ++ s.Equal("scale-job-terminal", released) + jobs, err := s.store.ListAllJobs(s.adminCtx) + s.Require().NoError(err) + s.Empty(jobs) + } + + func (s *PoolStressTestSuite) TestReconcileStartedScaleSetJobPreservesRunningQueueIntent() { ++ s.requireScaleSet() + released := false + reconciled := false +- nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, _ string) (bool, error) { ++ nddevRemoveAuthoritativeQueueIntent = func(context.Context, string) (bool, error) { + released = true + return true, nil + } +@@ -911,19 +1052,9 @@ func (s *PoolStressTestSuite) TestReconcileStartedScaleSetJobPreservesRunningQue + reconciled = got.ScaleSetJobID == "scale-job-started" && got.RepositoryOwner == "test-owner" && got.RepositoryName == "test-repo" + return true, nil + } +- _, err := s.store.CreateEntityScaleSet(s.adminCtx, s.entity, params.CreateScaleSetParams{ProviderName: "test-provider", Name: "nddev-linux-standard", Image: "test-image", Flavor: "test-flavor", MaxRunners: 1, OSType: commonParams.Linux, OSArch: commonParams.Amd64, Enabled: true}) +- s.Require().NoError(err) +- job := params.Job{ +- ScaleSetJobID: "scale-job-started", RunID: 4444, Name: "soak-fast-cpu-1", +- Status: string(params.JobStatusQueued), RepositoryOwner: "test-owner", RepositoryName: "test-repo", +- Labels: []string{"nddev-linux-standard"}, +- } +- _, err = s.store.CreateOrUpdateJob(s.adminCtx, job) +- s.Require().NoError(err) +- s.ghcliMock.EXPECT().ListWorkflowJobs(mock.Anything, "test-owner", "test-repo", int64(4444), mock.Anything). +- Return(&github.Jobs{Jobs: []*github.WorkflowJob{{ +- Name: github.Ptr("soak-fast-cpu-1"), Status: github.Ptr("in_progress"), +- }}}, &github.Response{}, nil).Once() ++ job := s.storeQueuedScaleSetJob("scale-job-started", 4444, "soak-fast-cpu-1") ++ sha := "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" ++ s.expectBoundScaleSetIdentity(4444, 9020, 601, 7001, 1, sha, "scale-job-started", "in_progress", "") + + s.True(s.mgr.reconcileStaleScaleSetJob(job)) + s.False(released, "a stale queued DB duplicate released the running capacity intent") +@@ -933,6 +1064,314 @@ func (s *PoolStressTestSuite) TestReconcileStartedScaleSetJobPreservesRunningQue + s.Empty(jobs) + } + ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsAmbiguousExternalID() { ++ job := s.storeQueuedScaleSetJob("scale-job-dup", 4700, "wanted") ++ sha := "ffffffffffffffffffffffffffffffffffffffff" ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4700)). ++ Return(scaleSetIdentityRun(4700, 9030, sha, 1), &github.Response{}, nil).Once() ++ page, resp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(701, "scale-job-dup", "queued", sha, 9030), ++ scaleSetIdentityCheckRun(702, "scale-job-dup", "completed", sha, 9030), ++ }, 2, 0) ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9030), mock.MatchedBy(allCheckRuns)). ++ Return(page, resp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) expectBoundScaleSetIdentity(runID, suiteID, checkID, jobID int64, attempt int, sha, guid, status, conclusion string) { ++ s.T().Helper() ++ listed := scaleSetIdentityJob(jobID, checkID, runID, int64(attempt), status, conclusion, sha, []string{"nddev-linux-standard"}) ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(checkID, guid, status, sha, suiteID), ++ }, 1, 0) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{listed}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", runID). ++ Return(scaleSetIdentityRun(runID, suiteID, sha, attempt), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", suiteID, mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", runID, int64(attempt), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.ghcliMock.EXPECT().GetWorkflowJobByID(mock.Anything, "test-owner", "test-repo", jobID). ++ Return(listed, &github.Response{}, nil).Once() ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobBindsDistinctCheckAndJobIDs() { ++ s.requireScaleSet() ++ released := "" ++ nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, jobID string) (bool, error) { ++ released = jobID ++ return true, nil ++ } ++ job := s.storeQueuedScaleSetJob("scale-job-distinct", 4800, "wanted") ++ sha := "1111111111111111111111111111111111111111" ++ s.expectBoundScaleSetIdentity(4800, 9040, 900, 800, 1, sha, "scale-job-distinct", "completed", "success") ++ s.True(s.mgr.reconcileStaleScaleSetJob(job)) ++ s.Equal("scale-job-distinct", released) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsWrongCheckRunURL() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-url", 4801, "wanted") ++ sha := "2222222222222222222222222222222222222222" ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(900, "scale-job-url", "completed", sha, 9041), ++ }, 1, 0) ++ wrong := scaleSetIdentityJob(800, 901, 4801, 1, "completed", "success", sha, []string{"nddev-linux-standard"}) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{wrong}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4801)). ++ Return(scaleSetIdentityRun(4801, 9041, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9041), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", int64(4801), int64(1), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsIncompleteCheckTotal() { ++ job := s.storeQueuedScaleSetJob("scale-job-total", 4802, "wanted") ++ sha := "3333333333333333333333333333333333333333" ++ page, resp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(101, "scale-job-total", "queued", sha, 9042), ++ }, 16, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4802)). ++ Return(scaleSetIdentityRun(4802, 9042, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9042), mock.MatchedBy(allCheckRuns)). ++ Return(page, resp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsMissingConclusionAndLabels() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-fields", 4803, "wanted") ++ sha := "4444444444444444444444444444444444444444" ++ s.expectBoundScaleSetIdentity(4803, 9043, 910, 810, 1, sha, "scale-job-fields", "completed", "") ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsMissingJobSHA() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-sha", 4804, "wanted") ++ sha := "5555555555555555555555555555555555555555" ++ listed := scaleSetIdentityJob(820, 920, 4804, 1, "completed", "success", "", []string{"nddev-linux-standard"}) ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(920, "scale-job-sha", "completed", sha, 9044), ++ }, 1, 0) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{listed}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4804)). ++ Return(scaleSetIdentityRun(4804, 9044, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9044), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", int64(4804), int64(1), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.ghcliMock.EXPECT().GetWorkflowJobByID(mock.Anything, "test-owner", "test-repo", int64(820)). ++ Return(listed, &github.Response{}, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsMissingLabels() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-labels", 4805, "wanted") ++ sha := "6666666666666666666666666666666666666666" ++ listed := scaleSetIdentityJob(821, 921, 4805, 1, "completed", "success", sha, nil) ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(921, "scale-job-labels", "completed", sha, 9045), ++ }, 1, 0) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{listed}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4805)). ++ Return(scaleSetIdentityRun(4805, 9045, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9045), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", int64(4805), int64(1), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.ghcliMock.EXPECT().GetWorkflowJobByID(mock.Anything, "test-owner", "test-repo", int64(821)). ++ Return(listed, &github.Response{}, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsWrongScaleSet() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-set", 4806, "wanted") ++ sha := "7777777777777777777777777777777777777777" ++ listed := scaleSetIdentityJob(822, 922, 4806, 1, "completed", "success", sha, []string{"other-scale-set"}) ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(922, "scale-job-set", "completed", sha, 9046), ++ }, 1, 0) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{listed}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4806)). ++ Return(scaleSetIdentityRun(4806, 9046, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9046), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", int64(4806), int64(1), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.ghcliMock.EXPECT().GetWorkflowJobByID(mock.Anything, "test-owner", "test-repo", int64(822)). ++ Return(listed, &github.Response{}, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsNonActionsProducer() { ++ job := s.storeQueuedScaleSetJob("scale-job-app", 4807, "wanted") ++ sha := "8888888888888888888888888888888888888888" ++ check := scaleSetIdentityCheckRun(923, "scale-job-app", "completed", sha, 9047) ++ check.App = &github.App{Slug: github.Ptr("dependabot")} ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{check}, 1, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4807)). ++ Return(scaleSetIdentityRun(4807, 9047, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9047), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsOmittedCheckTotal() { ++ job := s.storeQueuedScaleSetJob("scale-job-omit-total", 4809, "wanted") ++ sha := "abababababababababababababababababababab" ++ page := &github.ListCheckRunsResults{CheckRuns: []*github.CheckRun{ ++ scaleSetIdentityCheckRun(101, "scale-job-omit-total", "queued", sha, 9049), ++ }} ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4809)). ++ Return(scaleSetIdentityRun(4809, 9049, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9049), mock.MatchedBy(allCheckRuns)). ++ Return(page, &github.Response{}, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsChangingCheckTotal() { ++ job := s.storeQueuedScaleSetJob("scale-job-changing-total", 4810, "wanted") ++ sha := "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd" ++ page1, resp1 := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(101, "scale-job-changing-total", "queued", sha, 9050), ++ }, 2, 2) ++ page2, resp2 := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(102, "unrelated", "queued", sha, 9050), ++ }, 3, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4810)). ++ Return(scaleSetIdentityRun(4810, 9050, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9050), mock.MatchedBy(func(opts *github.ListCheckRunsOptions) bool { ++ return allCheckRuns(opts) && opts.Page == 0 ++ })).Return(page1, resp1, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9050), mock.MatchedBy(func(opts *github.ListCheckRunsOptions) bool { ++ return allCheckRuns(opts) && opts.Page == 2 ++ })).Return(page2, resp2, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobConcurrentCompletedAndUnknown() { ++ s.requireScaleSet() ++ var released atomic.Value ++ nddevRemoveAuthoritativeQueueIntent = func(_ context.Context, jobID string) (bool, error) { ++ released.Store(jobID) ++ return true, nil ++ } ++ unknown := s.storeQueuedScaleSetJob("scale-job-concurrent-unknown", 4811, "wanted") ++ terminal := s.storeQueuedScaleSetJob("scale-job-concurrent-terminal", 4812, "wanted") ++ unknownSHA := "1111111111111111111111111111111111111111" ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4811)). ++ Return(scaleSetIdentityRun(4811, 9051, unknownSHA, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9051), mock.MatchedBy(allCheckRuns)). ++ Return(&github.ListCheckRunsResults{CheckRuns: []*github.CheckRun{ ++ scaleSetIdentityCheckRun(111, "other-guid", "queued", unknownSHA, 9051), ++ }, Total: github.Ptr(1)}, &github.Response{}, nil).Once() ++ s.expectBoundScaleSetIdentity(4812, 9052, 930, 830, 1, "2222222222222222222222222222222222222222", "scale-job-concurrent-terminal", "completed", "success") ++ ++ var wg sync.WaitGroup ++ var unknownRetained, terminalDeleted bool ++ wg.Add(2) ++ go func() { ++ defer wg.Done() ++ unknownRetained = !s.mgr.reconcileStaleScaleSetJob(unknown) ++ }() ++ go func() { ++ defer wg.Done() ++ terminalDeleted = s.mgr.reconcileStaleScaleSetJob(terminal) ++ }() ++ wg.Wait() ++ s.True(unknownRetained) ++ s.True(terminalDeleted) ++ s.Equal("scale-job-concurrent-terminal", released.Load()) ++ jobs, err := s.store.ListAllJobs(s.adminCtx) ++ s.Require().NoError(err) ++ s.Require().Len(jobs, 1) ++ s.Equal("scale-job-concurrent-unknown", jobs[0].ScaleSetJobID) ++} ++ ++func (s *PoolStressTestSuite) TestReconcileStaleScaleSetJobRetainsIncompleteJobTotal() { ++ s.requireScaleSet() ++ job := s.storeQueuedScaleSetJob("scale-job-jobs", 4808, "wanted") ++ sha := "9999999999999999999999999999999999999999" ++ listed := scaleSetIdentityJob(824, 924, 4808, 1, "completed", "success", sha, []string{"nddev-linux-standard"}) ++ checkPage, checkResp := completeCheckPage([]*github.CheckRun{ ++ scaleSetIdentityCheckRun(924, "scale-job-jobs", "completed", sha, 9048), ++ }, 1, 0) ++ jobPage, jobResp := completeJobPage([]*github.WorkflowJob{listed}, 16, 0) ++ s.ghcliMock.EXPECT().GetWorkflowRunByID(mock.Anything, "test-owner", "test-repo", int64(4808)). ++ Return(scaleSetIdentityRun(4808, 9048, sha, 1), &github.Response{}, nil).Once() ++ s.ghcliMock.EXPECT().ListCheckRunsCheckSuite(mock.Anything, "test-owner", "test-repo", int64(9048), mock.MatchedBy(allCheckRuns)). ++ Return(checkPage, checkResp, nil).Once() ++ s.ghcliMock.EXPECT().ListWorkflowJobsAttempt(mock.Anything, "test-owner", "test-repo", int64(4808), int64(1), mock.Anything). ++ Return(jobPage, jobResp, nil).Once() ++ s.False(s.mgr.reconcileStaleScaleSetJob(job)) ++} ++ ++func TestGitHubListCursorRejectsDuplicatesAndMissingResponse(t *testing.T) { ++ cursor := newGitHubListCursor() ++ if _, ok := cursor.accept(0, nil, []int64{1}, github.Ptr(1)); ok { ++ t.Fatal("missing response must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ if _, ok := cursor.accept(0, &github.Response{}, []int64{1}, nil); ok { ++ t.Fatal("missing total must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ done, ok := cursor.accept(0, &github.Response{NextPage: 2}, []int64{1}, github.Ptr(2)) ++ if !ok || done { ++ t.Fatal("first page should continue") ++ } ++ if _, ok := cursor.accept(2, &github.Response{}, []int64{1}, github.Ptr(2)); ok { ++ t.Fatal("duplicate IDs must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ _, ok = cursor.accept(0, &github.Response{NextPage: 2}, []int64{1}, github.Ptr(2)) ++ if !ok { ++ t.Fatal("first page") ++ } ++ if _, ok := cursor.accept(2, &github.Response{NextPage: 2}, []int64{2}, github.Ptr(2)); ok { ++ t.Fatal("repeating next page must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ if _, ok := cursor.accept(0, &github.Response{NextPage: 2}, []int64{1}, github.Ptr(2)); !ok { ++ t.Fatal("first page") ++ } ++ if _, ok := cursor.accept(0, &github.Response{NextPage: 2}, []int64{2}, github.Ptr(2)); ok { ++ t.Fatal("duplicate page must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ if _, ok := cursor.accept(0, &github.Response{}, []int64{1}, github.Ptr(2)); ok { ++ t.Fatal("incomplete total must be incomplete") ++ } ++ cursor = newGitHubListCursor() ++ if _, ok := cursor.accept(0, &github.Response{NextPage: 2}, []int64{1}, github.Ptr(2)); !ok { ++ t.Fatal("first page") ++ } ++ if _, ok := cursor.accept(2, &github.Response{}, []int64{2}, github.Ptr(3)); ok { ++ t.Fatal("changing total must be incomplete") ++ } ++} ++ ++func TestWorkflowJobBindsCheckRunByURLPathID(t *testing.T) { ++ check := scaleSetIdentityCheckRun(12, "guid", "queued", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1) ++ job := scaleSetIdentityJob(800, 12, 1, 1, "queued", "", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil) ++ if !workflowJobBindsCheckRun(job, check) { ++ t.Fatal("matching check-run path ID must bind") ++ } ++ job.CheckRunURL = github.Ptr("https://api.github.com/repos/test-owner/test-repo/check-runs/2") ++ if workflowJobBindsCheckRun(job, check) { ++ t.Fatal("a shorter numeric suffix must not bind a different check ID") ++ } ++ job.CheckRunURL = github.Ptr("https://api.github.com/repos/test-owner/test-repo/check-runs/12?per_page=100") ++ if !workflowJobBindsCheckRun(job, check) { ++ t.Fatal("query string on check_run_url must still bind by path ID") ++ } ++} ++ + func mustParseUUID(s string) uuid.UUID { + u, err := uuid.Parse(s) + if err != nil { +diff --git a/runner/pool/scale_set_identity.go b/runner/pool/scale_set_identity.go +new file mode 100644 +index 0000000..204bc71 +--- /dev/null ++++ b/runner/pool/scale_set_identity.go +@@ -0,0 +1,175 @@ ++// Copyright 2026 Cloudbase Solutions SRL ++// ++// Licensed under the Apache License, Version 2.0 (the "License"); you may ++// not use this file except in compliance with the License. You may obtain ++// a copy of the License at ++// ++// http://www.apache.org/licenses/LICENSE-2.0 ++// ++// Unless required by applicable law or agreed to in writing, software ++// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++// License for the specific language governing permissions and limitations ++// under the License. ++ ++package pool ++ ++import ( ++ "fmt" ++ "net/url" ++ "strconv" ++ "strings" ++ ++ "github.com/google/go-github/v84/github" ++ ++ "github.com/cloudbase/garm/params" ++) ++ ++const ( ++ scaleSetListPageSize = 100 ++ scaleSetListMaxPages = 10 ++ githubActionsAppSlug = "github-actions" ++) ++ ++func checkRunIDFromURL(raw string) (int64, bool) { ++ parsed, err := url.Parse(raw) ++ if err != nil || parsed.Scheme == "" || parsed.Host == "" || parsed.Opaque != "" { ++ return 0, false ++ } ++ parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") ++ if len(parts) < 2 || parts[len(parts)-2] != "check-runs" { ++ return 0, false ++ } ++ id, err := strconv.ParseInt(parts[len(parts)-1], 10, 64) ++ if err != nil || id <= 0 { ++ return 0, false ++ } ++ return id, true ++} ++ ++func workflowJobBindsCheckRun(job *github.WorkflowJob, check *github.CheckRun) bool { ++ if job == nil || check == nil || check.GetID() == 0 { ++ return false ++ } ++ jobCheckID, ok := checkRunIDFromURL(job.GetCheckRunURL()) ++ if !ok || jobCheckID != check.GetID() { ++ return false ++ } ++ if check.GetURL() == "" { ++ return true ++ } ++ listedID, ok := checkRunIDFromURL(check.GetURL()) ++ return ok && listedID == check.GetID() ++} ++ ++func isGitHubActionsCheckRun(check *github.CheckRun) bool { ++ if check == nil || check.App == nil || check.App.GetSlug() == "" { ++ return false ++ } ++ return strings.EqualFold(check.App.GetSlug(), githubActionsAppSlug) ++} ++ ++func validTerminalConclusion(conclusion string) bool { ++ switch conclusion { ++ case "success", "failure", "cancelled", "skipped", "timed_out", "startup_failure": ++ return true ++ default: ++ return false ++ } ++} ++ ++func exactHeadSHA(values ...string) (string, bool) { ++ var sha string ++ for _, value := range values { ++ if value == "" { ++ return "", false ++ } ++ if sha == "" { ++ sha = value ++ continue ++ } ++ if !strings.EqualFold(sha, value) { ++ return "", false ++ } ++ } ++ return sha, sha != "" ++} ++ ++type githubListCursor struct { ++ seenPages map[int]struct{} ++ seenIDs map[int64]struct{} ++ collected int ++ expectedTotal int ++ hasTotal bool ++} ++ ++func newGitHubListCursor() githubListCursor { ++ return githubListCursor{ ++ seenPages: map[int]struct{}{}, ++ seenIDs: map[int64]struct{}{}, ++ } ++} ++ ++func (c *githubListCursor) accept(requestedPage int, resp *github.Response, ids []int64, total *int) (done bool, ok bool) { ++ if c == nil || resp == nil || total == nil || *total < 0 { ++ return false, false ++ } ++ if !c.hasTotal { ++ c.expectedTotal = *total ++ c.hasTotal = true ++ } else if *total != c.expectedTotal { ++ return false, false ++ } ++ if _, exists := c.seenPages[requestedPage]; exists { ++ return false, false ++ } ++ c.seenPages[requestedPage] = struct{}{} ++ for _, id := range ids { ++ if id == 0 { ++ return false, false ++ } ++ if _, exists := c.seenIDs[id]; exists { ++ return false, false ++ } ++ c.seenIDs[id] = struct{}{} ++ } ++ c.collected += len(ids) ++ if c.collected > c.expectedTotal { ++ return false, false ++ } ++ if resp.NextPage != 0 { ++ if resp.NextPage == requestedPage || resp.NextPage < 0 { ++ return false, false ++ } ++ if _, exists := c.seenPages[resp.NextPage]; exists { ++ return false, false ++ } ++ return false, true ++ } ++ if c.collected != c.expectedTotal { ++ return false, false ++ } ++ return true, true ++} ++ ++func exactScaleSetForJob(scaleSets []params.ScaleSet, job params.Job) (params.ScaleSet, error) { ++ if len(job.Labels) == 0 { ++ return params.ScaleSet{}, fmt.Errorf("queued job has no scale-set labels") ++ } ++ matches := make([]params.ScaleSet, 0, 1) ++ for _, scaleSet := range scaleSets { ++ if !scaleSet.Enabled { ++ continue ++ } ++ for _, label := range job.Labels { ++ if label == scaleSet.Name { ++ matches = append(matches, scaleSet) ++ break ++ } ++ } ++ } ++ if len(matches) != 1 { ++ return params.ScaleSet{}, fmt.Errorf("queued job matches %d enabled scale sets", len(matches)) ++ } ++ return matches[0], nil ++} +diff --git a/runner/pool/stub_client.go b/runner/pool/stub_client.go +index 3b41c24..3f3fb7c 100644 +--- a/runner/pool/stub_client.go ++++ b/runner/pool/stub_client.go +@@ -79,6 +79,14 @@ func (s *stubGithubClient) ListWorkflowJobs(_ context.Context, _, _ string, _ in + return nil, nil, s.err + } + ++func (s *stubGithubClient) ListWorkflowJobsAttempt(_ context.Context, _, _ string, _, _ int64, _ *github.ListOptions) (*github.Jobs, *github.Response, error) { ++ return nil, nil, s.err ++} ++ ++func (s *stubGithubClient) ListCheckRunsCheckSuite(_ context.Context, _, _ string, _ int64, _ *github.ListCheckRunsOptions) (*github.ListCheckRunsResults, *github.Response, error) { ++ return nil, nil, s.err ++} ++ + func (s *stubGithubClient) GetEntity() params.ForgeEntity { + return params.ForgeEntity{} + } +diff --git a/util/github/client.go b/util/github/client.go +index 530ad0f..e9286ab 100644 +--- a/util/github/client.go ++++ b/util/github/client.go +@@ -572,6 +572,52 @@ func (g *githubClient) RateLimit(ctx context.Context) (*github.RateLimits, error + return limits, nil + } + ++func (g *githubClient) ListCheckRunsCheckSuite(ctx context.Context, owner, repo string, checkSuiteID int64, opts *github.ListCheckRunsOptions) (ret *github.ListCheckRunsResults, response *github.Response, err error) { ++ metrics.GithubOperationCount.WithLabelValues( ++ "ListCheckRunsCheckSuite", ++ g.entity.LabelScope(), ++ ).Inc() ++ defer func() { ++ if err != nil { ++ metrics.GithubOperationFailedCount.WithLabelValues( ++ "ListCheckRunsCheckSuite", ++ g.entity.LabelScope(), ++ ).Inc() ++ } ++ }() ++ if g.cli == nil || g.cli.Checks == nil { ++ return nil, nil, errors.New("github checks client is unavailable") ++ } ++ ret, response, err = g.cli.Checks.ListCheckRunsCheckSuite(ctx, owner, repo, checkSuiteID, opts) ++ if err == nil && response != nil { ++ g.recordLimits(response.Rate) ++ } ++ return ret, response, err ++} ++ ++func (g *githubClient) ListWorkflowJobsAttempt(ctx context.Context, owner, repo string, runID, attemptNumber int64, opts *github.ListOptions) (ret *github.Jobs, response *github.Response, err error) { ++ metrics.GithubOperationCount.WithLabelValues( ++ "ListWorkflowJobsAttempt", ++ g.entity.LabelScope(), ++ ).Inc() ++ defer func() { ++ if err != nil { ++ metrics.GithubOperationFailedCount.WithLabelValues( ++ "ListWorkflowJobsAttempt", ++ g.entity.LabelScope(), ++ ).Inc() ++ } ++ }() ++ if g.ActionsService == nil { ++ return nil, nil, errors.New("github actions client is unavailable") ++ } ++ ret, response, err = g.ActionsService.ListWorkflowJobsAttempt(ctx, owner, repo, runID, attemptNumber, opts) ++ if err == nil && response != nil { ++ g.recordLimits(response.Rate) ++ } ++ return ret, response, err ++} ++ + func (g *githubClient) GetEntity() params.ForgeEntity { + return g.entity + } diff --git a/third_party/garm/patches/0033-live-message-demand-authority.patch b/third_party/garm/patches/0033-live-message-demand-authority.patch new file mode 100644 index 0000000..ae69ea9 --- /dev/null +++ b/third_party/garm/patches/0033-live-message-demand-authority.patch @@ -0,0 +1,1068 @@ +diff --git a/util/github/scalesets/client_testing.go b/util/github/scalesets/client_testing.go +new file mode 100644 +index 0000000..bb0000f +--- /dev/null ++++ b/util/github/scalesets/client_testing.go +@@ -0,0 +1,27 @@ ++//go:build testing ++ ++package scalesets ++ ++import ( ++ "net/http" ++ ++ "github.com/google/go-github/v84/github" ++ ++ "github.com/cloudbase/garm/params" ++ "github.com/cloudbase/garm/runner/common" ++) ++ ++func NewClientWithActionsService( ++ gh common.GithubClient, ++ httpClient *http.Client, ++ info params.ActionsServiceAdminInfoResponse, ++ token *github.RegistrationToken, ++) *ScaleSetClient { ++ infoCopy := info ++ return &ScaleSetClient{ ++ ghCli: gh, ++ httpClient: httpClient, ++ actionsServiceInfo: &infoCopy, ++ runnerRegistrationToken: token, ++ } ++} +diff --git a/util/github/scalesets/message_sessions.go b/util/github/scalesets/message_sessions.go +index b75890c..3fe40b0 100644 +--- a/util/github/scalesets/message_sessions.go ++++ b/util/github/scalesets/message_sessions.go +@@ -61,20 +61,63 @@ func (m *MessageSession) Close() error { + } + + func (m *MessageSession) MessageQueueAccessToken() string { +- return m.session.MessageQueueAccessToken ++ session := m.snapshot() ++ if session == nil { ++ return "" ++ } ++ return session.MessageQueueAccessToken ++} ++ ++func (m *MessageSession) snapshot() *params.RunnerScaleSetSession { ++ if m == nil { ++ return nil ++ } ++ m.mux.Lock() ++ defer m.mux.Unlock() ++ return m.session ++} ++ ++func (m *MessageSession) SessionID() string { ++ session := m.snapshot() ++ if session == nil || session.SessionID == nil { ++ return "" ++ } ++ return session.SessionID.String() ++} ++ ++func (m *MessageSession) Statistics() *params.RunnerScaleSetStatistic { ++ session := m.snapshot() ++ if session == nil { ++ return nil ++ } ++ return session.Statistics + } + + func (m *MessageSession) LastError() error { ++ if m == nil { ++ return nil ++ } ++ m.mux.Lock() ++ defer m.mux.Unlock() + return m.lastErr + } + ++func (m *MessageSession) setLastError(err error) { ++ m.mux.Lock() ++ m.lastErr = err ++ m.mux.Unlock() ++} ++ + func (m *MessageSession) loop() { +- slog.DebugContext(m.ctx, "starting message session refresh loop", "session_id", m.session.SessionID.String()) ++ slog.DebugContext(m.ctx, "starting message session refresh loop", "session_id", m.SessionID()) + timer := time.NewTicker(1 * time.Minute) + defer timer.Stop() + defer m.Close() + +- if m.closed { ++ m.mux.Lock() ++ closed := m.closed ++ m.mux.Unlock() ++ if closed { + slog.DebugContext(m.ctx, "message session refresh loop closed") + return + } +@@ -91,21 +134,30 @@ func (m *MessageSession) loop() { + // We endlessly retry. If it's a transient error, it should eventually + // work, if it's credentials issues, users can update them. + slog.With(slog.Any("error", err)).ErrorContext(m.ctx, "failed to refresh message queue token") +- m.lastErr = err ++ m.setLastError(err) + continue + } +- m.lastErr = nil ++ m.setLastError(nil) + } + } + } + + func (m *MessageSession) SessionsRelativeURL() (string, error) { ++ m.mux.Lock() ++ defer m.mux.Unlock() ++ return m.sessionsRelativeURLLocked() ++} ++ ++func (m *MessageSession) sessionsRelativeURLLocked() (string, error) { + if m.session == nil { + return "", fmt.Errorf("session is nil") + } + if m.session.RunnerScaleSet == nil { + return "", fmt.Errorf("runner scale set is nil") + } ++ if m.session.SessionID == nil { ++ return "", fmt.Errorf("session id is nil") ++ } + relativePath := fmt.Sprintf("%s/%d/sessions/%s", scaleSetEndpoint, m.session.RunnerScaleSet.ID, m.session.SessionID.String()) + return relativePath, nil + } +@@ -118,11 +170,11 @@ func (m *MessageSession) Refresh(ctx context.Context) (err error) { + } + }() + +- slog.DebugContext(ctx, "refreshing message session token", "session_id", m.session.SessionID.String()) ++ slog.DebugContext(ctx, "refreshing message session token", "session_id", m.SessionID()) + m.mux.Lock() + defer m.mux.Unlock() + +- relPath, err := m.SessionsRelativeURL() ++ relPath, err := m.sessionsRelativeURLLocked() + if err != nil { + return fmt.Errorf("failed to get session URL: %w", err) + } +@@ -146,11 +198,12 @@ func (m *MessageSession) Refresh(ctx context.Context) (err error) { + } + + func (m *MessageSession) maybeRefreshToken(ctx context.Context) error { +- if m.session == nil { ++ session := m.snapshot() ++ if session == nil { + return fmt.Errorf("session is nil") + } + +- expiresAt, err := m.session.ExiresAt() ++ expiresAt, err := session.ExiresAt() + if err != nil { + return fmt.Errorf("failed to get expires at: %w", err) + } +@@ -161,7 +214,7 @@ func (m *MessageSession) maybeRefreshToken(ctx context.Context) error { + } + expiresIn := time.Duration(randInt.Int64())*time.Second + 10*time.Minute + slog.DebugContext(ctx, "checking if message session token needs refresh", "expires_at", expiresAt) +- if m.session.ExpiresIn(expiresIn) { ++ if session.ExpiresIn(expiresIn) { + if err := m.Refresh(ctx); err != nil { + return fmt.Errorf("failed to refresh message queue token: %w", err) + } +@@ -178,7 +231,11 @@ func (m *MessageSession) GetMessage(ctx context.Context, lastMessageID int64, ma + } + }() + +- u, err := url.Parse(m.session.MessageQueueURL) ++ session := m.snapshot() ++ if session == nil { ++ return params.RunnerScaleSetMessage{}, fmt.Errorf("session is nil") ++ } ++ u, err := url.Parse(session.MessageQueueURL) + if err != nil { + return params.RunnerScaleSetMessage{}, err + } +@@ -195,7 +252,7 @@ func (m *MessageSession) GetMessage(ctx context.Context, lastMessageID int64, ma + } + + req.Header.Set("Accept", "application/json; api-version=6.0-preview") +- req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", m.session.MessageQueueAccessToken)) ++ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", session.MessageQueueAccessToken)) + req.Header.Set(maxCapacityHeader, fmt.Sprintf("%d", maxCapacity)) + + resp, err := m.ssCli.Do(req) +@@ -228,7 +285,11 @@ func (m *MessageSession) DeleteMessage(ctx context.Context, messageID int64) (er + } + }() + +- u, err := url.Parse(m.session.MessageQueueURL) ++ session := m.snapshot() ++ if session == nil { ++ return fmt.Errorf("session is nil") ++ } ++ u, err := url.Parse(session.MessageQueueURL) + if err != nil { + return err + } +@@ -241,7 +302,7 @@ func (m *MessageSession) DeleteMessage(ctx context.Context, messageID int64) (er + } + + req.Header.Set("Content-Type", "application/json") +- req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", m.session.MessageQueueAccessToken)) ++ req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", session.MessageQueueAccessToken)) + + resp, err := m.ssCli.Do(req) + if err != nil { +diff --git a/util/github/scalesets/message_sessions_race_test.go b/util/github/scalesets/message_sessions_race_test.go +new file mode 100644 +index 0000000..7cc66a5 +--- /dev/null ++++ b/util/github/scalesets/message_sessions_race_test.go +@@ -0,0 +1,64 @@ ++package scalesets ++ ++import ( ++ "context" ++ "errors" ++ "sync" ++ "testing" ++ ++ "github.com/google/uuid" ++ ++ "github.com/cloudbase/garm/params" ++) ++ ++func TestMessageSessionAccessorsRaceWithReplacement(t *testing.T) { ++ first := uuid.MustParse("00000000-0000-4000-8000-000000000001") ++ second := uuid.MustParse("00000000-0000-4000-8000-000000000002") ++ sess := &MessageSession{ ++ session: ¶ms.RunnerScaleSetSession{ ++ SessionID: &first, ++ MessageQueueURL: "https://example.invalid/queue", ++ MessageQueueAccessToken: "token-one", ++ Statistics: ¶ms.RunnerScaleSetStatistic{TotalAssignedJobs: 1}, ++ RunnerScaleSet: ¶ms.RunnerScaleSet{ID: 9}, ++ }, ++ ctx: context.Background(), ++ done: make(chan struct{}), ++ } ++ var wg sync.WaitGroup ++ for i := 0; i < 8; i++ { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for n := 0; n < 400; n++ { ++ _ = sess.SessionID() ++ _ = sess.Statistics() ++ _ = sess.MessageQueueAccessToken() ++ _ = sess.LastError() ++ _, _ = sess.SessionsRelativeURL() ++ } ++ }() ++ } ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for n := 0; n < 400; n++ { ++ id := first ++ if n%2 == 1 { ++ id = second ++ } ++ next := id ++ sess.mux.Lock() ++ sess.session = ¶ms.RunnerScaleSetSession{ ++ SessionID: &next, ++ MessageQueueURL: "https://example.invalid/queue", ++ MessageQueueAccessToken: "token-two", ++ Statistics: ¶ms.RunnerScaleSetStatistic{TotalAssignedJobs: n % 3}, ++ RunnerScaleSet: ¶ms.RunnerScaleSet{ID: 9}, ++ } ++ sess.lastErr = errors.New("refresh") ++ sess.mux.Unlock() ++ } ++ }() ++ wg.Wait() ++} +diff --git a/workers/scaleset/interfaces.go b/workers/scaleset/interfaces.go +index 0cc6526..e287e3e 100644 +--- a/workers/scaleset/interfaces.go ++++ b/workers/scaleset/interfaces.go +@@ -14,6 +14,8 @@ + package scaleset + + import ( ++ "time" ++ + "github.com/cloudbase/garm/params" + "github.com/cloudbase/garm/util/github/scalesets" + ) +@@ -24,6 +26,8 @@ type scaleSetHelper interface { + GetScaleSetClient() (*scalesets.ScaleSetClient, error) + SetLastMessageID(id int64) error + SetDesiredRunnerCount(count int) error ++ ObserveLiveMessageDemand(sessionID string, assigned int, messageID int64, observedAt time.Time) error ++ InvalidateLiveMessageDemand() + Owner() string + HandleJobsCompleted(jobs []params.ScaleSetJobMessage) error + HandleJobsStarted(jobs []params.ScaleSetJobMessage) error +diff --git a/workers/scaleset/listener_session_test.go b/workers/scaleset/listener_session_test.go +new file mode 100644 +index 0000000..d4ae347 +--- /dev/null ++++ b/workers/scaleset/listener_session_test.go +@@ -0,0 +1,136 @@ ++//go:build testing ++ ++package scaleset ++ ++import ( ++ "context" ++ "encoding/base64" ++ "encoding/json" ++ "errors" ++ "fmt" ++ "net/http" ++ "net/http/httptest" ++ "strings" ++ "sync/atomic" ++ "testing" ++ "time" ++ ++ "github.com/google/go-github/v84/github" ++ "github.com/google/uuid" ++ ++ "github.com/cloudbase/garm/params" ++ "github.com/cloudbase/garm/runner/common/mocks" ++ "github.com/cloudbase/garm/util/github/scalesets" ++) ++ ++func unsignedExpJWT(exp time.Time) string { ++ header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"HS256","typ":"JWT"}`)) ++ payload := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"exp":%d}`, exp.Unix()))) ++ return header + "." + payload + ".sig" ++} ++ ++func TestListenerStartCleansSessionWhenObserveFails(t *testing.T) { ++ sessionID := uuid.MustParse("00000000-0000-4000-8000-000000000094") ++ var deleted atomic.Int32 ++ server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, req *http.Request) { ++ switch { ++ case req.Method == http.MethodPost && strings.HasSuffix(strings.TrimRight(req.URL.Path, "/"), "/sessions"): ++ body := params.RunnerScaleSetSession{ ++ SessionID: &sessionID, ++ OwnerName: "example-owner", ++ MessageQueueURL: req.Host, ++ MessageQueueAccessToken: unsignedExpJWT(time.Now().Add(time.Hour)), ++ RunnerScaleSet: ¶ms.RunnerScaleSet{ID: 9, Name: "example"}, ++ Statistics: ¶ms.RunnerScaleSetStatistic{TotalAssignedJobs: 0}, ++ } ++ writer.Header().Set("Content-Type", "application/json") ++ if err := json.NewEncoder(writer).Encode(body); err != nil { ++ t.Errorf("encode session: %v", err) ++ } ++ case req.Method == http.MethodDelete: ++ deleted.Add(1) ++ writer.WriteHeader(http.StatusOK) ++ default: ++ http.NotFound(writer, req) ++ } ++ })) ++ t.Cleanup(server.Close) ++ ++ gh := mocks.NewGithubClient(t) ++ gh.EXPECT().GetEntity().Return(params.ForgeEntity{ ++ EntityType: params.ForgeEntityTypeRepository, ++ Owner: "example-owner", ++ Name: "example-repository", ++ }).Maybe() ++ client := scalesets.NewClientWithActionsService( ++ gh, ++ server.Client(), ++ params.ActionsServiceAdminInfoResponse{ ++ URL: server.URL + "/", ++ Token: unsignedExpJWT(time.Now().Add(time.Hour)), ++ }, ++ &github.RegistrationToken{ ++ Token: github.Ptr("registration"), ++ ExpiresAt: &github.Timestamp{Time: time.Now().Add(time.Hour)}, ++ }, ++ ) ++ helper := &startFailureHelper{ ++ scaleSet: params.ScaleSet{ID: 1, ScaleSetID: 9, Name: "example", MaxRunners: 1, Enabled: true}, ++ client: client, ++ observe: errors.New("observe failed"), ++ } ++ now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ listener := newListener(context.Background(), helper) ++ listener.intentCoordinator = testQueueCoordinator(t, &now, nil) ++ if err := listener.Start(); err == nil { ++ t.Fatal("Start must fail when live demand observation fails") ++ } ++ if deleted.Load() < 1 { ++ t.Fatal("failed Start leaked the newly created message session") ++ } ++ if helper.invalidated < 2 { ++ t.Fatalf("start and failed adoption must invalidate demand, got %d", helper.invalidated) ++ } ++ if listener.IsRunning() || listener.messageSession != nil { ++ t.Fatal("failed Start left a live listener session") ++ } ++ select { ++ case <-listener.listenerCtx.Done(): ++ default: ++ t.Fatal("failed Start leaked the listener context") ++ } ++} ++ ++type startFailureHelper struct { ++ scaleSet params.ScaleSet ++ client *scalesets.ScaleSetClient ++ observe error ++ invalidated int ++} ++ ++func (s *startFailureHelper) GetScaleSet() params.ScaleSet { return s.scaleSet } ++func (s *startFailureHelper) GetEntity() params.ForgeEntity { ++ return params.ForgeEntity{} ++} ++func (s *startFailureHelper) GetScaleSetClient() (*scalesets.ScaleSetClient, error) { ++ return s.client, nil ++} ++func (s *startFailureHelper) SetLastMessageID(id int64) error { ++ s.scaleSet.LastMessageID = id ++ return nil ++} ++func (s *startFailureHelper) SetDesiredRunnerCount(int) error { return nil } ++func (s *startFailureHelper) ObserveLiveMessageDemand(string, int, int64, time.Time) error { ++ return s.observe ++} ++func (s *startFailureHelper) InvalidateLiveMessageDemand() { s.invalidated++ } ++func (s *startFailureHelper) Owner() string { return "example-owner" } ++func (s *startFailureHelper) HandleJobsCompleted([]params.ScaleSetJobMessage) error { ++ return nil ++} ++func (s *startFailureHelper) HandleJobsStarted([]params.ScaleSetJobMessage) error { ++ return nil ++} ++func (s *startFailureHelper) HandleJobsAvailable([]params.ScaleSetJobMessage) error { ++ return nil ++} +diff --git a/workers/scaleset/message_demand_test.go b/workers/scaleset/message_demand_test.go +new file mode 100644 +index 0000000..1545f4e +--- /dev/null ++++ b/workers/scaleset/message_demand_test.go +@@ -0,0 +1,372 @@ ++package scaleset ++ ++import ( ++ "context" ++ "encoding/json" ++ "errors" ++ "sync" ++ "testing" ++ "time" ++ ++ dbCommon "github.com/cloudbase/garm/database/common" ++ "github.com/cloudbase/garm/params" ++ "github.com/cloudbase/garm/util/github/scalesets" ++) ++ ++type recordingScaleSetHelper struct { ++ scaleSet params.ScaleSet ++ observed []liveMessageDemand ++ invalidated int ++ started []params.ScaleSetJobMessage ++ desiredCount []int ++} ++ ++func (r *recordingScaleSetHelper) GetScaleSet() params.ScaleSet { return r.scaleSet } ++func (r *recordingScaleSetHelper) GetEntity() params.ForgeEntity { ++ return params.ForgeEntity{} ++} ++func (r *recordingScaleSetHelper) GetScaleSetClient() (*scalesets.ScaleSetClient, error) { ++ return nil, errors.New("recording helper does not open a scale set client") ++} ++func (r *recordingScaleSetHelper) SetLastMessageID(id int64) error { ++ r.scaleSet.LastMessageID = id ++ return nil ++} ++func (r *recordingScaleSetHelper) SetDesiredRunnerCount(count int) error { ++ r.desiredCount = append(r.desiredCount, count) ++ r.scaleSet.DesiredRunnerCount = count ++ return nil ++} ++func (r *recordingScaleSetHelper) ObserveLiveMessageDemand(sessionID string, assigned int, messageID int64, observedAt time.Time) error { ++ r.observed = append(r.observed, liveMessageDemand{ ++ sessionID: sessionID, ++ assigned: assigned, ++ observed: true, ++ messageID: messageID, ++ observedAt: observedAt, ++ }) ++ return r.SetDesiredRunnerCount(assigned) ++} ++func (r *recordingScaleSetHelper) InvalidateLiveMessageDemand() { r.invalidated++ } ++func (r *recordingScaleSetHelper) Owner() string { return "example-owner" } ++func (r *recordingScaleSetHelper) HandleJobsCompleted([]params.ScaleSetJobMessage) error { ++ return nil ++} ++func (r *recordingScaleSetHelper) HandleJobsStarted(jobs []params.ScaleSetJobMessage) error { ++ r.started = append(r.started, jobs...) ++ return nil ++} ++func (r *recordingScaleSetHelper) HandleJobsAvailable([]params.ScaleSetJobMessage) error { ++ return nil ++} ++ ++func testListener(helper scaleSetHelper, sessionID string) *scaleSetListener { ++ l := newListener(context.Background(), helper) ++ l.sessionID = sessionID ++ l.running.Store(true) ++ l.quit = make(chan struct{}) ++ ctx, cancel := context.WithCancel(context.Background()) ++ l.listenerCtx = ctx ++ l.cancelFunc = cancel ++ return l ++} ++ ++func jobMessage(messageType, jobID string, assigned, running int) params.RunnerScaleSetMessage { ++ body, err := json.Marshal([]params.ScaleSetJobMessage{{ ++ MessageType: messageType, ++ JobID: jobID, ++ JobDisplayName: "example-job", ++ RunnerName: "example-runner", ++ WorkflowRunID: 1, ++ }}) ++ if err != nil { ++ panic(err) ++ } ++ return params.RunnerScaleSetMessage{ ++ MessageID: 7, ++ MessageType: string(params.MessageTypeRunnerScaleSetJobMessages), ++ Body: string(body), ++ Statistics: ¶ms.RunnerScaleSetStatistic{TotalAssignedJobs: assigned, TotalRunningJobs: running}, ++ } ++} ++ ++func TestListenerObservesMessageCountUpdateAndZero(t *testing.T) { ++ helper := &recordingScaleSetHelper{scaleSet: params.ScaleSet{ID: 1, ScaleSetID: 9, Name: "example"}} ++ l := testListener(helper, "sess-live") ++ l.handleSessionMessage(jobMessage(params.MessageTypeJobAssigned, "job-a", 3, 0)) ++ if len(helper.observed) != 1 || helper.observed[0].sessionID != "sess-live" || helper.observed[0].assigned != 3 || helper.observed[0].messageID != 7 { ++ t.Fatalf("count update: %#v", helper.observed) ++ } ++ if helper.scaleSet.DesiredRunnerCount != 3 { ++ t.Fatalf("persisted count = %d, want 3", helper.scaleSet.DesiredRunnerCount) ++ } ++ l.handleSessionMessage(jobMessage(params.MessageTypeJobCompleted, "job-a", 0, 0)) ++ if len(helper.observed) != 2 || helper.observed[1].assigned != 0 || helper.invalidated != 0 { ++ t.Fatalf("live zero must stay observed on the current session: observed=%#v invalidated=%d", helper.observed, helper.invalidated) ++ } ++} ++ ++func TestListenerMissingStatisticsInvalidatesDemand(t *testing.T) { ++ helper := &recordingScaleSetHelper{scaleSet: params.ScaleSet{ID: 1, ScaleSetID: 9}} ++ l := testListener(helper, "sess-live") ++ l.handleSessionMessage(params.RunnerScaleSetMessage{ ++ MessageID: 8, ++ MessageType: string(params.MessageTypeRunnerScaleSetJobMessages), ++ Body: `[]`, ++ }) ++ if helper.invalidated != 1 || len(helper.observed) != 0 { ++ t.Fatalf("missing statistics must not become a zero snapshot: observed=%#v invalidated=%d", helper.observed, helper.invalidated) ++ } ++} ++ ++func TestListenerStopAndReplacedSessionInvalidateStaleZero(t *testing.T) { ++ helper := &recordingScaleSetHelper{scaleSet: params.ScaleSet{ID: 1, ScaleSetID: 9, DesiredRunnerCount: 0}} ++ l := testListener(helper, "sess-old") ++ l.handleSessionMessage(jobMessage(params.MessageTypeJobAssigned, "job-a", 0, 0)) ++ if err := l.Stop(); err != nil { ++ t.Fatal(err) ++ } ++ if helper.invalidated != 1 { ++ t.Fatalf("disconnected session must invalidate live demand, got %d", helper.invalidated) ++ } ++ replaced := testListener(helper, "sess-new") ++ if len(helper.observed) != 1 || helper.observed[0].sessionID != "sess-old" { ++ t.Fatalf("replacement without a new observation must not revive the old zero: %#v", helper.observed) ++ } ++ replaced.handleSessionMessage(jobMessage(params.MessageTypeJobAssigned, "job-b", 2, 0)) ++ if got := helper.observed[len(helper.observed)-1]; got.sessionID != "sess-new" || got.assigned != 2 { ++ t.Fatalf("replaced session observation: %#v", helper.observed) ++ } ++} ++ ++func TestListenerJobStartedMessageObservesLiveDemand(t *testing.T) { ++ helper := &recordingScaleSetHelper{scaleSet: params.ScaleSet{ID: 1, ScaleSetID: 9}} ++ l := testListener(helper, "sess-live") ++ l.handleSessionMessage(jobMessage(params.MessageTypeJobStarted, "job-started", 1, 1)) ++ if len(helper.observed) != 1 || helper.observed[0].assigned != 1 || helper.observed[0].sessionID != "sess-live" { ++ t.Fatalf("JobStarted message must publish live demand before later client work: %#v", helper.observed) ++ } ++} ++ ++func TestWorkerLiveDemandIgnoresStaleDatabaseZero(t *testing.T) { ++ w := &Worker{ ++ ctx: context.Background(), ++ scaleSet: params.ScaleSet{ID: 4, DesiredRunnerCount: 0}, ++ } ++ if demand := w.liveMessageDemand(); demand.observed { ++ t.Fatal("unstarted worker must not treat a persisted zero as live demand") ++ } ++ w.liveDemand = liveMessageDemand{sessionID: "sess-old", assigned: 0, observed: true} ++ w.InvalidateLiveMessageDemand() ++ if w.liveMessageDemand().observed { ++ t.Fatal("disconnected session left stale zero as authority") ++ } ++ now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ got := evaluateIdleRetirement(idleRetirementEvidence{ ++ AgentID: 1, Name: "n", ScaleSetID: 1, LocalStatus: params.RunnerIdle, ++ CreatedAt: now.Add(-time.Hour), IdleCount: 2, Now: now, StatisticsPresent: true, ++ MessageDemandKnown: false, AssignedJobs: w.scaleSet.DesiredRunnerCount, ++ }) ++ if got.Eligible || got.Reason != "message-demand-unknown" { ++ t.Fatalf("stale DB zero must not retire idle runners: %#v", got) ++ } ++} ++ ++func TestHandleScaleSetEventDoesNotRestoreInvalidatedDemand(t *testing.T) { ++ w := &Worker{ ++ ctx: context.Background(), ++ scaleSet: params.ScaleSet{Enabled: true, MaxRunners: 2}, ++ autoscaleWake: make(chan struct{}, 1), ++ liveDemand: liveMessageDemand{}, ++ } ++ w.handleScaleSetEvent(dbCommon.ChangePayload{ ++ Operation: dbCommon.UpdateOperation, ++ Payload: params.ScaleSet{Enabled: true, MaxRunners: 2, DesiredRunnerCount: 0}, ++ }) ++ if w.liveMessageDemand().observed { ++ t.Fatal("database watcher zero must not become live message demand") ++ } ++} ++ ++func TestWorkerLiveDemandConcurrencyWithIdleAndScale(t *testing.T) { ++ w := &Worker{ ++ ctx: context.Background(), ++ scaleSet: params.ScaleSet{Enabled: true, MaxRunners: 8, Name: "example"}, ++ autoscaleWake: make(chan struct{}, 8), ++ } ++ helper := &forwardingDemandHelper{worker: w, scaleSet: w.scaleSet} ++ now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ l := testListener(helper, "sess-live") ++ var wg sync.WaitGroup ++ for i := 0; i < 8; i++ { ++ wg.Add(1) ++ go func() { ++ defer wg.Done() ++ for n := 0; n < 80; n++ { ++ switch n % 5 { ++ case 0: ++ l.handleSessionMessage(jobMessage(params.MessageTypeJobAssigned, "job-a", n%3, 0)) ++ case 1: ++ w.InvalidateLiveMessageDemand() ++ case 2: ++ demand := w.liveMessageDemand() ++ _, _ = w.demandGate.target(context.Background(), now, demand, w.scaleSet, 0, 4) ++ case 3: ++ demand := w.liveMessageDemand() ++ ev := baseEvidence(now) ++ ev.MessageDemandKnown = demand.idleFresh(now) ++ ev.AssignedJobs = demand.assigned ++ _ = evaluateIdleRetirement(ev) ++ default: ++ _ = w.liveMessageDemand().unchangedIdleZero(w.liveMessageDemand(), now) ++ } ++ } ++ }() ++ } ++ wg.Wait() ++} ++ ++type forwardingDemandHelper struct { ++ worker *Worker ++ scaleSet params.ScaleSet ++} ++ ++func (f *forwardingDemandHelper) GetScaleSet() params.ScaleSet { return f.scaleSet } ++func (f *forwardingDemandHelper) GetEntity() params.ForgeEntity { ++ return params.ForgeEntity{} ++} ++func (f *forwardingDemandHelper) GetScaleSetClient() (*scalesets.ScaleSetClient, error) { ++ return nil, errors.New("concurrency helper does not open a scale set client") ++} ++func (f *forwardingDemandHelper) SetLastMessageID(id int64) error { ++ f.scaleSet.LastMessageID = id ++ return nil ++} ++func (f *forwardingDemandHelper) SetDesiredRunnerCount(count int) error { ++ f.scaleSet.DesiredRunnerCount = count ++ return nil ++} ++func (f *forwardingDemandHelper) ObserveLiveMessageDemand(sessionID string, assigned int, messageID int64, observedAt time.Time) error { ++ return f.worker.ObserveLiveMessageDemand(sessionID, assigned, messageID, observedAt) ++} ++func (f *forwardingDemandHelper) InvalidateLiveMessageDemand() { ++ f.worker.InvalidateLiveMessageDemand() ++} ++func (f *forwardingDemandHelper) Owner() string { return "example-owner" } ++func (f *forwardingDemandHelper) HandleJobsCompleted([]params.ScaleSetJobMessage) error { ++ return nil ++} ++func (f *forwardingDemandHelper) HandleJobsStarted([]params.ScaleSetJobMessage) error { ++ return nil ++} ++func (f *forwardingDemandHelper) HandleJobsAvailable([]params.ScaleSetJobMessage) error { ++ return nil ++} ++ ++func TestSessionCreateZeroIsNotIdleAuthority(t *testing.T) { ++ w := &Worker{ctx: context.Background()} ++ now := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ if err := w.ObserveLiveMessageDemand("sess-live", 0, 0, now); err != nil { ++ t.Fatal(err) ++ } ++ demand := w.liveMessageDemand() ++ if !demand.scalingKnown() || demand.idleFresh(now) { ++ t.Fatalf("session-create zero must scale but not idle-retire: %#v", demand) ++ } ++ ev := baseEvidence(now) ++ ev.MessageDemandKnown = demand.idleFresh(now) ++ ev.AssignedJobs = demand.assigned ++ got := evaluateIdleRetirement(ev) ++ if got.Eligible || got.Reason != "message-demand-unknown" { ++ t.Fatalf("session-create zero retired idle capacity: %#v", got) ++ } ++} ++ ++func TestReplayedMessageIDDoesNotRefreshIdleFreshness(t *testing.T) { ++ w := &Worker{ctx: context.Background()} ++ t0 := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ if err := w.ObserveLiveMessageDemand("sess-live", 0, 9, t0); err != nil { ++ t.Fatal(err) ++ } ++ first := w.liveMessageDemand() ++ if !first.idleFresh(t0) || first.generation != 1 || first.messageID != 9 { ++ t.Fatalf("initial MESSAGE zero must be idle-fresh: %#v", first) ++ } ++ ++ replayAt := t0.Add(5 * time.Minute) ++ if err := w.ObserveLiveMessageDemand("sess-live", 0, 9, replayAt); err != nil { ++ t.Fatal(err) ++ } ++ replayed := w.liveMessageDemand() ++ if !replayed.observedAt.Equal(first.observedAt) || replayed.generation != first.generation || ++ replayed.messageID != first.messageID || replayed.assigned != first.assigned { ++ t.Fatalf("replayed messageID refreshed demand: first=%#v replayed=%#v", first, replayed) ++ } ++ if replayed.idleFresh(replayAt) { ++ t.Fatal("a replayed MESSAGE must not look freshly idle five minutes later") ++ } ++ ++ ev := baseEvidence(replayAt) ++ ev.MessageDemandKnown = replayed.idleFresh(replayAt) ++ ev.AssignedJobs = replayed.assigned ++ ev.DemandSessionID = replayed.sessionID ++ ev.DemandMessageID = replayed.messageID ++ ev.DemandGeneration = replayed.generation ++ got := evaluateIdleRetirement(ev) ++ if got.Eligible || got.Reason != "message-demand-unknown" { ++ t.Fatalf("replayed idle zero authorized retirement: %#v", got) ++ } ++} ++ ++func TestNewMessageIDRefreshesIdleObservation(t *testing.T) { ++ w := &Worker{ctx: context.Background()} ++ t0 := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ if err := w.ObserveLiveMessageDemand("sess-live", 0, 9, t0); err != nil { ++ t.Fatal(err) ++ } ++ next := t0.Add(30 * time.Second) ++ if err := w.ObserveLiveMessageDemand("sess-live", 0, 10, next); err != nil { ++ t.Fatal(err) ++ } ++ demand := w.liveMessageDemand() ++ if demand.messageID != 10 || !demand.observedAt.Equal(next) || demand.generation != 2 { ++ t.Fatalf("a new messageID must be a new observation: %#v", demand) ++ } ++ if !demand.idleFresh(next) { ++ t.Fatal("a new MESSAGE zero must be idle-fresh") ++ } ++} ++ ++func TestStaleSessionMessageDoesNotReplaceLiveDemand(t *testing.T) { ++ w := &Worker{ctx: context.Background()} ++ t0 := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ if err := w.ObserveLiveMessageDemand("sess-new", 2, 10, t0); err != nil { ++ t.Fatal(err) ++ } ++ first := w.liveMessageDemand() ++ if err := w.ObserveLiveMessageDemand("sess-old", 0, 11, t0.Add(time.Minute)); err != nil { ++ t.Fatal(err) ++ } ++ got := w.liveMessageDemand() ++ if got.sessionID != first.sessionID || got.messageID != first.messageID || ++ got.assigned != first.assigned || got.generation != first.generation || ++ !got.observedAt.Equal(first.observedAt) { ++ t.Fatalf("replaced session overwrote live demand: first=%#v got=%#v", first, got) ++ } ++} ++ ++func TestInvalidateAllowsReplacementSessionDemand(t *testing.T) { ++ w := &Worker{ctx: context.Background()} ++ t0 := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) ++ if err := w.ObserveLiveMessageDemand("sess-old", 0, 9, t0); err != nil { ++ t.Fatal(err) ++ } ++ w.InvalidateLiveMessageDemand() ++ if err := w.ObserveLiveMessageDemand("sess-new", 3, 1, t0.Add(time.Second)); err != nil { ++ t.Fatal(err) ++ } ++ got := w.liveMessageDemand() ++ if got.sessionID != "sess-new" || got.assigned != 3 || got.messageID != 1 { ++ t.Fatalf("replacement session was refused: %#v", got) ++ } ++} +diff --git a/workers/scaleset/scaleset.go b/workers/scaleset/scaleset.go +index 9883ec9..8b18cd8 100644 +--- a/workers/scaleset/scaleset.go ++++ b/workers/scaleset/scaleset.go +@@ -91,6 +91,8 @@ type Worker struct { + autoscaleWake chan struct{} + demandGate confirmedDemandGate + idleRetire idleRetirementGate ++ demandMu sync.Mutex ++ liveDemand liveMessageDemand + } + + func (w *Worker) ensureScaleSetInGitHub() error { +@@ -178,6 +180,7 @@ func (w *Worker) Stop() error { + close(w.quit) + } + w.listener.Stop() ++ w.InvalidateLiveMessageDemand() + return nil + } + +@@ -670,6 +673,7 @@ func (w *Worker) handleScaleSetEvent(event dbCommon.ChangePayload) { + if err := w.listener.Stop(); err != nil { + slog.ErrorContext(w.ctx, "error stopping listener", "error", err) + } ++ w.InvalidateLiveMessageDemand() + } + w.scaleSet = scaleSet + w.mux.Unlock() +@@ -891,7 +895,7 @@ func (w *Worker) handleScaleUp() { + slog.ErrorContext(w.ctx, "error getting scale set client", "error", err) + return + } +- confirmed, err := w.demandGate.target(w.ctx, time.Now(), scaleSetCli, w.scaleSet, current, target) ++ confirmed, err := w.demandGate.target(w.ctx, time.Now(), w.liveMessageDemand(), w.scaleSet, current, target) + if err != nil { + slog.ErrorContext(w.ctx, "current GitHub demand could not be confirmed; retaining queued jobs", "error", err) + return +diff --git a/workers/scaleset/scaleset_helper.go b/workers/scaleset/scaleset_helper.go +index 449180e..ffe6a46 100644 +--- a/workers/scaleset/scaleset_helper.go ++++ b/workers/scaleset/scaleset_helper.go +@@ -18,6 +18,7 @@ import ( + "fmt" + "log/slog" + "strings" ++ "time" + + runnerErrors "github.com/cloudbase/garm-provider-common/errors" + commonParams "github.com/cloudbase/garm-provider-common/params" +@@ -203,3 +204,57 @@ func (w *Worker) SetDesiredRunnerCount(count int) error { + } + return nil + } ++ ++func (w *Worker) ObserveLiveMessageDemand(sessionID string, assigned int, messageID int64, observedAt time.Time) error { ++ if sessionID == "" { ++ return fmt.Errorf("message session identity is missing") ++ } ++ if assigned < 0 || observedAt.IsZero() { ++ return fmt.Errorf("message demand is invalid") ++ } ++ w.demandMu.Lock() ++ current := w.liveDemand ++ replay := current.observed && current.sessionID == sessionID && current.messageID == messageID ++ staleSession := current.observed && current.sessionID != "" && current.sessionID != sessionID ++ w.demandMu.Unlock() ++ if replay || staleSession { ++ // A retained MESSAGE is not a heartbeat. A late delivery from a replaced ++ // session must not become the current idle/scaling authority. ++ return nil ++ } ++ if w.store != nil { ++ if err := w.SetDesiredRunnerCount(assigned); err != nil { ++ return err ++ } ++ } ++ w.demandMu.Lock() ++ defer w.demandMu.Unlock() ++ current = w.liveDemand ++ if current.observed && current.sessionID == sessionID && current.messageID == messageID { ++ return nil ++ } ++ if current.observed && current.sessionID != "" && current.sessionID != sessionID { ++ return nil ++ } ++ w.liveDemand = liveMessageDemand{ ++ sessionID: sessionID, ++ assigned: assigned, ++ observed: true, ++ messageID: messageID, ++ observedAt: observedAt, ++ generation: current.generation + 1, ++ } ++ return nil ++} ++ ++func (w *Worker) InvalidateLiveMessageDemand() { ++ w.demandMu.Lock() ++ w.liveDemand = liveMessageDemand{generation: w.liveDemand.generation + 1} ++ w.demandMu.Unlock() ++} ++ ++func (w *Worker) liveMessageDemand() liveMessageDemand { ++ w.demandMu.Lock() ++ defer w.demandMu.Unlock() ++ return w.liveDemand ++} +diff --git a/workers/scaleset/scaleset_listener.go b/workers/scaleset/scaleset_listener.go +index a689316..be8910d 100644 +--- a/workers/scaleset/scaleset_listener.go ++++ b/workers/scaleset/scaleset_listener.go +@@ -59,6 +59,7 @@ type scaleSetListener struct { + + scaleSetHelper scaleSetHelper + messageSession *scalesets.MessageSession ++ sessionID string + // intentCoordinator is the single durable admission point before + // AcquireJobs. Every scale-set listener shares its fsync journal and lock. + intentCoordinator *queueIntentCoordinator +@@ -83,7 +84,10 @@ func (l *scaleSetListener) Start() error { + if err := l.intentCoordinator.Validate(); err != nil { + return fmt.Errorf("validating queue-intent coordinator: %w", err) + } ++ l.scaleSetHelper.InvalidateLiveMessageDemand() ++ l.sessionID = "" + ++ adopted := false + listenCtx, listenCancelFunc := context.WithCancel(context.Background()) + listenCtx = garmUtil.CopySlogValuesToNewCtx(l.ctx, listenCtx) + listenCtx = garmUtil.WithSlogContext( +@@ -92,6 +96,26 @@ func (l *scaleSetListener) Start() error { + ) + l.listenerCtx = listenCtx + l.cancelFunc = listenCancelFunc ++ var session *scalesets.MessageSession ++ defer func() { ++ if adopted { ++ return ++ } ++ listenCancelFunc() ++ if session == nil { ++ return ++ } ++ _ = session.Close() ++ scaleSetClient, clientErr := l.scaleSetHelper.GetScaleSetClient() ++ if clientErr != nil { ++ slog.ErrorContext(l.ctx, "getting scale set client to delete unstarted session", "error", clientErr) ++ } else if err := scaleSetClient.DeleteMessageSession(context.Background(), session); err != nil { ++ slog.ErrorContext(l.ctx, "deleting unstarted message session", "error", err) ++ } ++ l.messageSession = nil ++ l.sessionID = "" ++ l.scaleSetHelper.InvalidateLiveMessageDemand() ++ }() + + scaleSet := l.scaleSetHelper.GetScaleSet() + scaleSetClient, err := l.scaleSetHelper.GetScaleSetClient() +@@ -99,7 +123,7 @@ func (l *scaleSetListener) Start() error { + return fmt.Errorf("getting scale set client: %w", err) + } + slog.DebugContext(l.ctx, "creating new message session", "scale_set", scaleSet.ScaleSetID) +- session, err := scaleSetClient.CreateMessageSession( ++ session, err = scaleSetClient.CreateMessageSession( + l.listenerCtx, scaleSet.ScaleSetID, + l.scaleSetHelper.Owner(), + ) +@@ -107,11 +131,23 @@ func (l *scaleSetListener) Start() error { + return fmt.Errorf("creating message session: %w", err) + } + l.messageSession = session ++ l.sessionID = session.SessionID() ++ if l.sessionID == "" { ++ return fmt.Errorf("message session identity is missing") ++ } ++ if stats := session.Statistics(); stats != nil { ++ // Session-create statistics are the latest assigned counter for ++ // ordinary scaling (messageID 0). They are not idle-retirement ++ // evidence; idleFresh requires a later MESSAGE with messageID > 0. ++ if err := l.scaleSetHelper.ObserveLiveMessageDemand(l.sessionID, stats.TotalAssignedJobs, 0, time.Now().UTC()); err != nil { ++ return fmt.Errorf("observing session demand: %w", err) ++ } ++ } + l.quit = make(chan struct{}) + l.running.Store(true) + l.loopExited = make(chan struct{}) + go l.loop() +- ++ adopted = true + return nil + } + +@@ -122,6 +158,8 @@ func (l *scaleSetListener) Stop() error { + if !l.running.Load() { + return nil + } ++ l.scaleSetHelper.InvalidateLiveMessageDemand() ++ l.sessionID = "" + if l.messageSession != nil { + slog.DebugContext(l.ctx, "closing message session", "scale_set", l.scaleSetHelper.GetScaleSet().ScaleSetID) + if err := l.messageSession.Close(); err != nil { +@@ -187,6 +225,15 @@ func (l *scaleSetListener) handleSessionMessage(msg params.RunnerScaleSetMessage + } + } + ++ if msg.Statistics == nil || l.sessionID == "" { ++ slog.WarnContext(l.ctx, "message demand is missing session statistics; retaining unknown authority") ++ l.scaleSetHelper.InvalidateLiveMessageDemand() ++ } else if err := l.scaleSetHelper.ObserveLiveMessageDemand(l.sessionID, msg.Statistics.TotalAssignedJobs, msg.MessageID, time.Now().UTC()); err != nil { ++ slog.ErrorContext(l.ctx, "observing live message demand", "error", err) ++ l.scaleSetHelper.InvalidateLiveMessageDemand() ++ return ++ } ++ + scaleSetClient, err := l.scaleSetHelper.GetScaleSetClient() + if err != nil { + slog.ErrorContext(l.ctx, "getting scale set client", "error", err) +@@ -282,9 +329,6 @@ func (l *scaleSetListener) handleSessionMessage(msg params.RunnerScaleSetMessage + } + } + +- if err := l.scaleSetHelper.SetDesiredRunnerCount(msg.Statistics.TotalAssignedJobs); err != nil { +- slog.ErrorContext(l.ctx, "setting desired runner count", "error", err) +- } + if retainAvailableMessage { + slog.DebugContext(l.ctx, "retaining unacquired available-job message", "message_id", msg.MessageID) + timer := time.NewTimer(time.Second) +@@ -352,7 +396,8 @@ func (l *scaleSetListener) loop() { + retryAfterUnauthorized = false + if !msg.IsNil() { + // Longpoll returns after 50 seconds. If no message arrives during that interval +- // we get a nil message. We can simply ignore it and continue. ++ // we get a nil/202 message. That is not a heartbeat and must not refresh ++ // idle-retirement freshness or invent a zero assigned count. + slog.DebugContext(l.ctx, "handling message", "message_id", msg.MessageID) + l.handleSessionMessage(msg) + } diff --git a/third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch b/third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch new file mode 100644 index 0000000..6cd46bd --- /dev/null +++ b/third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch @@ -0,0 +1,288 @@ +diff --git a/go.mod b/go.mod +index 0fe0c49..5811cde 100644 +--- a/go.mod ++++ b/go.mod +@@ -28,9 +28,9 @@ require ( + github.com/spf13/cobra v1.10.2 + github.com/stretchr/testify v1.11.1 + golang.org/x/crypto v0.50.0 +- golang.org/x/mod v0.35.0 ++ golang.org/x/mod v0.37.0 + golang.org/x/oauth2 v0.36.0 +- golang.org/x/sync v0.20.0 ++ golang.org/x/sync v0.21.0 + golang.org/x/term v0.42.0 + gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 +@@ -99,7 +99,7 @@ require ( + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sys v0.43.0 // indirect +- golang.org/x/text v0.36.0 // indirect ++ golang.org/x/text v0.39.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + ) +diff --git a/go.sum b/go.sum +index c45535b..bdad52a 100644 +--- a/go.sum ++++ b/go.sum +@@ -215,6 +215,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 + golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= + golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= + golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= ++golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= ++golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= + golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= + golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +@@ -228,6 +230,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ + golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= + golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= + golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= ++golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= ++golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= + golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= + golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= + golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +@@ -250,6 +254,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= + golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= + golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= + golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= ++golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= ++golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= + golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= + golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= + golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go +index f69fd75..c261a8e 100644 +--- a/vendor/golang.org/x/sync/errgroup/errgroup.go ++++ b/vendor/golang.org/x/sync/errgroup/errgroup.go +@@ -109,7 +109,7 @@ func (g *Group) TryGo(f func() error) bool { + if g.sem != nil { + select { + case g.sem <- token{}: +- // Note: this allows barging iff channels in general allow barging. ++ // Note: this allows barging if and only if channels in general allow barging. + default: + return false + } +diff --git a/vendor/golang.org/x/text/cases/context.go b/vendor/golang.org/x/text/cases/context.go +index e9aa9e1..a28f45d 100644 +--- a/vendor/golang.org/x/text/cases/context.go ++++ b/vendor/golang.org/x/text/cases/context.go +@@ -249,7 +249,7 @@ func upper(c *context) bool { + return c.copy() + } + +-// isUpper writes the isUppercase version of the current rune to dst. ++// isUpper reports whether the current rune is in upper case. + func isUpper(c *context) bool { + ct := c.caseType() + if c.info&hasMappingMask == 0 || ct == cUpper { +diff --git a/vendor/golang.org/x/text/cases/map.go b/vendor/golang.org/x/text/cases/map.go +index 0f7c6a1..51a6830 100644 +--- a/vendor/golang.org/x/text/cases/map.go ++++ b/vendor/golang.org/x/text/cases/map.go +@@ -774,7 +774,7 @@ func nlTitle(c *context) bool { + // From CLDR: + // # Special titlecasing for Dutch initial "ij". + // ::Any-Title(); +- // # Fix up Ij at the beginning of a "word" (per Any-Title, notUAX #29) ++ // # Fix up Ij at the beginning of a "word" (per Any-Title, not UAX #29) + // [:^WB=ALetter:] [:WB=Extend:]* [[:WB=MidLetter:][:WB=MidNumLet:]]? { Ij } → IJ ; + if c.src[c.pSrc] != 'I' && c.src[c.pSrc] != 'i' { + return title(c) +@@ -794,7 +794,7 @@ func nlTitleSpan(c *context) bool { + // From CLDR: + // # Special titlecasing for Dutch initial "ij". + // ::Any-Title(); +- // # Fix up Ij at the beginning of a "word" (per Any-Title, notUAX #29) ++ // # Fix up Ij at the beginning of a "word" (per Any-Title, not UAX #29) + // [:^WB=ALetter:] [:WB=Extend:]* [[:WB=MidLetter:][:WB=MidNumLet:]]? { Ij } → IJ ; + if c.src[c.pSrc] != 'I' { + return isTitle(c) +diff --git a/vendor/golang.org/x/text/unicode/norm/forminfo.go b/vendor/golang.org/x/text/unicode/norm/forminfo.go +index f3a234e..b3cf5d9 100644 +--- a/vendor/golang.org/x/text/unicode/norm/forminfo.go ++++ b/vendor/golang.org/x/text/unicode/norm/forminfo.go +@@ -121,8 +121,12 @@ func (p Properties) BoundaryAfter() bool { + // + // When all 6 bits are zero, the character is inert, meaning it is never + // influenced by normalization. ++// ++// We set flags to 0x80 (high bit 7 unused in quick check data) to indicate an invalid rune. + type qcInfo uint8 + ++func (p Properties) isInvalid() bool { return p.flags == 0x80 } ++ + func (p Properties) isYesC() bool { return p.flags&0x10 == 0 } + func (p Properties) isYesD() bool { return p.flags&0x4 == 0 } + +@@ -247,6 +251,9 @@ func (f Form) PropertiesString(s string) Properties { + // to a Properties. See the comment at the top of the file + // for more information on the format. + func compInfo(v uint16, sz int) Properties { ++ if sz == 0 { ++ return Properties{flags: 0x80, size: 1} ++ } + if v == 0 { + return Properties{size: uint8(sz)} + } else if v >= 0x8000 { +@@ -254,7 +261,7 @@ func compInfo(v uint16, sz int) Properties { + size: uint8(sz), + ccc: uint8(v), + tccc: uint8(v), +- flags: qcInfo(v >> 8), ++ flags: qcInfo(v>>8) & 0x3f, + } + if p.ccc > 0 || p.combinesBackward() { + p.nLead = uint8(p.flags & 0x3) +diff --git a/vendor/golang.org/x/text/unicode/norm/iter.go b/vendor/golang.org/x/text/unicode/norm/iter.go +index 417c6b2..3cc0592 100644 +--- a/vendor/golang.org/x/text/unicode/norm/iter.go ++++ b/vendor/golang.org/x/text/unicode/norm/iter.go +@@ -376,16 +376,12 @@ func nextComposed(i *Iter) []byte { + goto doNorm + } + prevCC = i.info.tccc +- sz := int(i.info.size) +- if sz == 0 { +- sz = 1 // illegal rune: copy byte-by-byte +- } +- p := outp + sz ++ p := outp + int(i.info.size) + if p > len(i.buf) { + break + } + outp = p +- i.p += sz ++ i.p += int(i.info.size) + if i.p >= i.rb.nsrc { + i.setDone() + break +diff --git a/vendor/golang.org/x/text/unicode/norm/normalize.go b/vendor/golang.org/x/text/unicode/norm/normalize.go +index 4747ad0..60b1511 100644 +--- a/vendor/golang.org/x/text/unicode/norm/normalize.go ++++ b/vendor/golang.org/x/text/unicode/norm/normalize.go +@@ -148,7 +148,7 @@ func (f Form) IsNormalString(s string) bool { + // patched buffer and whether the decomposition is still in progress. + func patchTail(rb *reorderBuffer) bool { + info, p := lastRuneStart(&rb.f, rb.out) +- if p == -1 || info.size == 0 { ++ if p == -1 || info.isInvalid() { + return true + } + end := p + int(info.size) +@@ -225,7 +225,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte { + } + fd := &rb.f + if doMerge { +- var info Properties ++ info := Properties{flags: 0x80, size: 1} // invalid rune + if p < n { + info = fd.info(src, p) + if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 { +@@ -235,7 +235,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte { + p = decomposeSegment(rb, p, true) + } + } +- if info.size == 0 { ++ if info.isInvalid() { + rb.doFlush() + // Append incomplete UTF-8 encoding. + return src.appendSlice(rb.out, p, n) +@@ -314,7 +314,7 @@ func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) + continue + } + info := f.info(src, i) +- if info.size == 0 { ++ if info.isInvalid() { + if atEOF { + // include incomplete runes + return n, true +@@ -379,7 +379,7 @@ func (f Form) firstBoundary(src input, nsrc int) int { + // CGJ insertion points correctly. Luckily it doesn't have to. + for { + info := fd.info(src, i) +- if info.size == 0 { ++ if info.isInvalid() { + return -1 + } + if s := ss.next(info); s != ssSuccess { +@@ -424,7 +424,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int { + } + fd := formTable[f] + info := fd.info(src, 0) +- if info.size == 0 { ++ if info.isInvalid() { + if atEOF { + return 1 + } +@@ -435,7 +435,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int { + + for i := int(info.size); i < nsrc; i += int(info.size) { + info = fd.info(src, i) +- if info.size == 0 { ++ if info.isInvalid() { + if atEOF { + return i + } +@@ -465,7 +465,7 @@ func lastBoundary(fd *formInfo, b []byte) int { + if p == -1 { + return -1 + } +- if info.size == 0 { // ends with incomplete rune ++ if info.isInvalid() { // ends with incomplete rune + if p == 0 { // starts with incomplete rune + return -1 + } +@@ -504,7 +504,7 @@ func lastBoundary(fd *formInfo, b []byte) int { + func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int { + // Force one character to be consumed. + info := rb.f.info(rb.src, sp) +- if info.size == 0 { ++ if info.isInvalid() { + return 0 + } + if s := rb.ss.next(info); s == ssStarter { +@@ -528,7 +528,7 @@ func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int { + break + } + info = rb.f.info(rb.src, sp) +- if info.size == 0 { ++ if info.isInvalid() { + if !atEOF { + return int(iShortSrc) + } +diff --git a/vendor/modules.txt b/vendor/modules.txt +index b43d43d..e0ca9e7 100644 +--- a/vendor/modules.txt ++++ b/vendor/modules.txt +@@ -344,7 +344,7 @@ golang.org/x/crypto/chacha20poly1305 + golang.org/x/crypto/hkdf + golang.org/x/crypto/internal/alias + golang.org/x/crypto/internal/poly1305 +-# golang.org/x/mod v0.35.0 ++# golang.org/x/mod v0.37.0 + ## explicit; go 1.25.0 + golang.org/x/mod/semver + # golang.org/x/net v0.53.0 +@@ -356,7 +356,7 @@ golang.org/x/net/proxy + ## explicit; go 1.25.0 + golang.org/x/oauth2 + golang.org/x/oauth2/internal +-# golang.org/x/sync v0.20.0 ++# golang.org/x/sync v0.21.0 + ## explicit; go 1.25.0 + golang.org/x/sync/errgroup + # golang.org/x/sys v0.43.0 +@@ -368,7 +368,7 @@ golang.org/x/sys/windows + # golang.org/x/term v0.42.0 + ## explicit; go 1.25.0 + golang.org/x/term +-# golang.org/x/text v0.36.0 ++# golang.org/x/text v0.39.0 + ## explicit; go 1.25.0 + golang.org/x/text/cases + golang.org/x/text/encoding