From 72409b4dab404cd4e0839c56152c1edd3820294f Mon Sep 17 00:00:00 2001 From: Lukas Frank Date: Tue, 25 Aug 2026 12:07:14 +0200 Subject: [PATCH 1/4] `MachinePool` lifecycle docs Signed-off-by: Lukas Frank --- docs/.vitepress/config.mts | 1 + .../architecture/machine-pool-lifecycle.md | 88 +++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 docs/iaas/architecture/machine-pool-lifecycle.md diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 4d03a52..ade981a 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -98,6 +98,7 @@ export default withMermaid({ collapsed: true, items: [ { text: 'MachinePool Health', link: '/iaas/architecture/machine-pool-health' }, + { text: 'MachinePool Lifecycle', link: '/iaas/architecture/machine-pool-lifecycle' }, ], }, { text: 'Networking', link: '/iaas/architecture/networking' }, diff --git a/docs/iaas/architecture/machine-pool-lifecycle.md b/docs/iaas/architecture/machine-pool-lifecycle.md new file mode 100644 index 0000000..918a56c --- /dev/null +++ b/docs/iaas/architecture/machine-pool-lifecycle.md @@ -0,0 +1,88 @@ +# MachinePool Lifecycle + +The `pool-lifecycle-controller` coordinates [Cluster API](https://cluster-api.sigs.k8s.io/) (CAPI) +node deletion with graceful eviction of the IronCore `Machine`s running on the affected pool. +A CAPI compute node backs an IronCore `MachinePool`, and the IronCore VMs of that pool run on it. +So when CAPI replaces or deletes such a node for instance during a rolling upgrade or scale-down, +deleting it while those VMs are still running would drop their workloads. This +controller holds the node deletion until the pool has been drained. + +It builds directly on the [Machine Eviction](/iaas/architecture/machine-eviction) mechanism: +draining is performed by tainting the `MachinePool` with a `NoExecute` maintenance taint and waiting +for the eviction to complete. + +The controller reconciles the external CAPI `Machine` together with IronCore `MachinePool`/`Machine`. + +A CAPI `Machine` is linked to the `MachinePool` it backs through its `status.nodeRef.name`, which +matches the `MachinePool` name. + +## The Pre-Drain Hook + +CAPI supports [pre-drain lifecycle hooks](https://cluster-api.sigs.k8s.io/tasks/experimental-features/lifecycle-hooks): +an annotation on a CAPI `Machine` that pauses deletion before the node is drained until the +annotation is removed. The controller uses this to interpose IronCore eviction. + +For every CAPI `Machine` matching the configured selector, the controller ensures two things while +the machine is alive: + +```yaml +metadata: + finalizers: + - maintenance.ironcore.dev/machinepool-cleanup + annotations: + pre-drain.delete.hook.machine.cluster.x-k8s.io/ironcore-maintenance: ironcore-maintenance +``` + +- The **finalizer** keeps the controller in the loop when the CAPI `Machine` is deleted, so it can + run its cleanup before the object disappears. +- The **pre-drain hook** blocks CAPI from draining and removing the node until the controller + clears it. + +Which CAPI `Machine`s are managed is restricted by the `--capi-machine-selector` label selector +(empty selects all). + +## Lifecycle Flow + +When CAPI decides to delete a `Machine` (setting its `deletionTimestamp`), it stalls at the +pre-drain hook and the controller takes over: + +1. The controller resolves the `MachinePool` from the CAPI `Machine`'s `status.nodeRef.name`. +2. It ensures the maintenance taints on that `MachinePool`, adding both effects under the key + `maintenance.ironcore.dev`: + + ```yaml + spec: + taints: + - key: maintenance.ironcore.dev + effect: NoSchedule + - key: maintenance.ironcore.dev + effect: NoExecute + ``` + + `NoSchedule` stops new machines from landing on the pool while it is draining; `NoExecute` + triggers [eviction](/iaas/architecture/machine-pool-eviction) of every bound `Machine` that does + not tolerate the taint. +3. It checks whether any non-tolerating `Machine`s are still bound to the pool. As long as some + remain, it **holds the pre-drain hook** and requeues. The node stays up while VMs shut down + gracefully. +4. Once the pool is drained (only tolerating machines, if any, remain), the controller **removes the + pre-drain hook**, allowing CAPI to drain the Kubernetes node and delete it. +5. After CAPI has removed its own `Machine` finalizer, the controller **deletes the now-empty + `MachinePool`** object. +6. Finally, it removes its own finalizer from the CAPI `Machine`, letting the object be garbage + collected. + +If a CAPI `Machine` has no `status.nodeRef` (it never became a node), there is nothing to drain: the +controller simply clears the pre-drain hook and its finalizer. + + +## Relationship to Eviction and Health + +This controller is the automation layer that turns an infrastructure-level node deletion into an +ordered IronCore drain. It leans on two lower-level mechanisms: + +- [Machine Eviction](/iaas/architecture/machine-eviction) provides the `NoExecute` taint + semantics and the per-`Machine` graceful shutdown that the controller drives. +- [MachinePool Health](/iaas/architecture/machine-pool-health) provides the pool status the broader + system relies on; note that this controller reacts to *planned* CAPI node deletions, not to a pool + becoming unhealthy on its own. From 7c823589b07b7f62c07f6ce56b3a0b97a8f9d88f Mon Sep 17 00:00:00 2001 From: Lukas Frank Date: Tue, 25 Aug 2026 12:14:40 +0200 Subject: [PATCH 2/4] fix dead link Signed-off-by: Lukas Frank --- docs/iaas/architecture/machine-pool-lifecycle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/iaas/architecture/machine-pool-lifecycle.md b/docs/iaas/architecture/machine-pool-lifecycle.md index 918a56c..050a92a 100644 --- a/docs/iaas/architecture/machine-pool-lifecycle.md +++ b/docs/iaas/architecture/machine-pool-lifecycle.md @@ -60,7 +60,7 @@ pre-drain hook and the controller takes over: ``` `NoSchedule` stops new machines from landing on the pool while it is draining; `NoExecute` - triggers [eviction](/iaas/architecture/machine-pool-eviction) of every bound `Machine` that does + triggers [eviction](/iaas/architecture/machine-eviction) of every bound `Machine` that does not tolerate the taint. 3. It checks whether any non-tolerating `Machine`s are still bound to the pool. As long as some remain, it **holds the pre-drain hook** and requeues. The node stays up while VMs shut down From 351419cfed488b5ebf4ac700b988039035748c58 Mon Sep 17 00:00:00 2001 From: Lukas Frank Date: Tue, 25 Aug 2026 16:56:58 +0200 Subject: [PATCH 3/4] PR Review Signed-off-by: Lukas Frank --- docs/.vitepress/config.mts | 2 +- docs/iaas/architecture/machine-pool-lifecycle.md | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index e9c8084..b36f4f8 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -99,7 +99,7 @@ export default withMermaid({ items: [ { text: 'MachinePool Health', link: '/iaas/architecture/machine-pool-health' }, { text: 'Machine Eviction', link: '/iaas/architecture/machine-eviction' }, - { text: 'MachinePool Lifecycle', link: '/iaas/architecture/machine-pool-lifecycle' }, + { text: 'MachinePool Lifecycle (CAPI)', link: '/iaas/architecture/machine-pool-lifecycle' }, ], }, { text: 'Networking', link: '/iaas/architecture/networking' }, diff --git a/docs/iaas/architecture/machine-pool-lifecycle.md b/docs/iaas/architecture/machine-pool-lifecycle.md index 050a92a..49cbcec 100644 --- a/docs/iaas/architecture/machine-pool-lifecycle.md +++ b/docs/iaas/architecture/machine-pool-lifecycle.md @@ -1,7 +1,9 @@ -# MachinePool Lifecycle +# CAPI-based MachinePool Lifecycle + +This page describes MachinePool lifecycle management for [Cluster API](https://cluster-api.sigs.k8s.io/) +(CAPI) based cluster setups. The generic drain primitive it relies on the taint-based eviction that +works independently of CAPI and is documented in [Machine Eviction](/iaas/architecture/machine-eviction). -The `pool-lifecycle-controller` coordinates [Cluster API](https://cluster-api.sigs.k8s.io/) (CAPI) -node deletion with graceful eviction of the IronCore `Machine`s running on the affected pool. A CAPI compute node backs an IronCore `MachinePool`, and the IronCore VMs of that pool run on it. So when CAPI replaces or deletes such a node for instance during a rolling upgrade or scale-down, deleting it while those VMs are still running would drop their workloads. This @@ -18,7 +20,7 @@ matches the `MachinePool` name. ## The Pre-Drain Hook -CAPI supports [pre-drain lifecycle hooks](https://cluster-api.sigs.k8s.io/tasks/experimental-features/lifecycle-hooks): +CAPI supports [pre-drain lifecycle hooks](https://cluster-api.sigs.k8s.io/reference/api/labels-and-annotations.html?highlight=pre-drain.delete.hook.machine.cluster.x-k8s.io#supported-annotations): an annotation on a CAPI `Machine` that pauses deletion before the node is drained until the annotation is removed. The controller uses this to interpose IronCore eviction. From 688f424182ec0835ac23055d6bd892248c0796d4 Mon Sep 17 00:00:00 2001 From: Lukas Frank Date: Wed, 26 Aug 2026 11:42:30 +0200 Subject: [PATCH 4/4] PR Review Signed-off-by: Lukas Frank --- docs/.vitepress/config.mts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index b36f4f8..e9c8084 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -99,7 +99,7 @@ export default withMermaid({ items: [ { text: 'MachinePool Health', link: '/iaas/architecture/machine-pool-health' }, { text: 'Machine Eviction', link: '/iaas/architecture/machine-eviction' }, - { text: 'MachinePool Lifecycle (CAPI)', link: '/iaas/architecture/machine-pool-lifecycle' }, + { text: 'MachinePool Lifecycle', link: '/iaas/architecture/machine-pool-lifecycle' }, ], }, { text: 'Networking', link: '/iaas/architecture/networking' },