Skip to content

feat: Add skip-provisioning maintenance annotation for Device controller #482

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nikatza wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: Add skip-provisioning maintenance annotation for Device controller #482

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/core/v1alpha1/groupversion_info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,11 @@ const (
// device-side operations. This is useful for recovering from terminal states (e.g., Failed)
// after manual intervention.
DeviceMaintenanceResetPhase = "reset-phase"
// DeviceMaintenanceSkipProvisioning is for devices provisioned out-of-band.

@felix-kaestner felix-kaestner Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
// DeviceMaintenanceSkipProvisioning is for devices provisioned out-of-band.
// DeviceMaintenanceSkipProvisioning is for devices provisioned outside of the operator.

The out-of-band could be a bit misleading w/ out-of-band connectivity.

// Transitions the device from Pending (or Provisioning) directly to Running, but only if
// spec.provisioning is defined. The annotation is removed once a phase transition is triggered by
// the annotation itself; otherwise it is ignored and left on the resource.
DeviceMaintenanceSkipProvisioning = "skip-provisioning"
)

// Condition types that are used across different objects.
Expand Down
23 changes: 23 additions & 0 deletions internal/controller/core/device_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,14 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
return ctrl.Result{}, reconcile.TerminalError(errors.New("provider does not support provisioning"))
}

if action, ok := obj.Annotations[v1alpha1.DeviceMaintenanceAnnotation]; ok && action == v1alpha1.DeviceMaintenanceSkipProvisioning {
// Skip provisioning if the device is annotated to skip provisioning.
obj.Status.Phase = v1alpha1.DevicePhaseRunning
r.Recorder.Eventf(obj, nil, "Normal", "SkipProvisioning", "Maintenance", "Device in pending phase will skip provisioning due to maintenance annotation")
delete(obj.Annotations, v1alpha1.DeviceMaintenanceAnnotation)
return ctrl.Result{}, nil
}

log.Info("Device is in pending phase, starting provisioning")
conditions.Set(obj, metav1.Condition{
Type: v1alpha1.ReadyCondition,
Expand All @@ -149,6 +157,17 @@ func (r *DeviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ c
return ctrl.Result{}, nil

case v1alpha1.DevicePhaseProvisioning:
if action, ok := obj.Annotations[v1alpha1.DeviceMaintenanceAnnotation]; ok && action == v1alpha1.DeviceMaintenanceSkipProvisioning {
// Close all active provisioning attempts and skip provisioning if the device is annotated accordingly.
if activeProv := obj.GetActiveProvisioning(); activeProv != nil {
activeProv.EndTime = metav1.Now()
}
obj.Status.Phase = v1alpha1.DevicePhaseRunning
r.Recorder.Eventf(obj, nil, "Normal", "SkipProvisioning", "Maintenance", "Device in provisioning phase will skip provisioning due to maintenance annotation")
delete(obj.Annotations, v1alpha1.DeviceMaintenanceAnnotation)
return ctrl.Result{}, nil
}

if obj.Spec.Provisioning == nil {
log.Info("Provisioning configuration was removed, resetting device into pending phase")
if activeProv := obj.GetActiveProvisioning(); activeProv != nil {
Expand Down Expand Up @@ -429,6 +448,10 @@ func (r *DeviceReconciler) reconcileMaintenance(ctx context.Context, obj *v1alph
}

switch action {
case v1alpha1.DeviceMaintenanceSkipProvisioning:
// The annotation must stay present until the annotation is consumed.

@felix-kaestner felix-kaestner Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this mean that when a Device is in phase==Running this annotation will stay there forever?

return nil

case v1alpha1.DeviceMaintenanceResetPhase:
// Reset phase is a soft reset that only changes the device phase to Pending without
// performing any device-side operations. This is useful for recovering from terminal
Expand Down
306 changes: 226 additions & 80 deletions internal/controller/core/device_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,86 +388,6 @@ var _ = Describe("Device Controller", func() {
}).Should(Succeed())
})

It("Should transition from Running to Provisioning once the reset-phase annotation is set", func() {
By("Creating a Device")
device := &v1alpha1.Device{
ObjectMeta: metav1.ObjectMeta{
Name: key.Name,
Namespace: key.Namespace,
},
Spec: v1alpha1.DeviceSpec{
Endpoint: v1alpha1.Endpoint{
Address: "192.168.10.5:9339",
SecretRef: &v1alpha1.SecretReference{
Name: name,
},
},
Provisioning: &v1alpha1.Provisioning{
BootScript: v1alpha1.TemplateSource{
Inline: new("boot nxos.bin"),
},
Image: v1alpha1.Image{
URL: "https://best-vendor-images.to/windows98",
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
ChecksumType: v1alpha1.ChecksumTypeMD5,
},
},
},
}
Expect(k8sClient.Create(ctx, device)).To(Succeed())

By("Verifying the device transitions to Provisioning phase")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
g.Expect(resource.Status.Conditions).To(HaveLen(3))
g.Expect(resource.Status.Conditions[0].Type).To(Equal(v1alpha1.ReadyCondition))
g.Expect(resource.Status.Conditions[1].Type).To(Equal(v1alpha1.PausedCondition))
g.Expect(resource.Status.Conditions[1].Status).To(Equal(metav1.ConditionFalse))
g.Expect(resource.Status.Conditions[2].Type).To(Equal(v1alpha1.ReachableCondition))
g.Expect(resource.Status.Conditions[2].Status).To(Equal(metav1.ConditionUnknown))
}).Should(Succeed())

By("Setting the device to Running phase")
orig := device.DeepCopy()
device.Status.Phase = v1alpha1.DevicePhaseRunning
Expect(k8sClient.Status().Patch(ctx, device, client.MergeFrom(orig))).To(Succeed())

By("Verifying the device transitions to Running phase")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseRunning))
g.Expect(resource.Status.Conditions).To(HaveLen(3))
g.Expect(resource.Status.Conditions[0].Type).To(Equal(v1alpha1.ReadyCondition))
g.Expect(resource.Status.Conditions[1].Type).To(Equal(v1alpha1.PausedCondition))
g.Expect(resource.Status.Conditions[1].Status).To(Equal(metav1.ConditionFalse))
g.Expect(resource.Status.Conditions[2].Type).To(Equal(v1alpha1.ReachableCondition))
g.Expect(resource.Status.Conditions[2].Status).To(Equal(metav1.ConditionTrue))
}).Should(Succeed())

By("Adding the reset-phase annotation to the device")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
patch := resource.DeepCopy()
annotations := make(map[string]string)
annotations[v1alpha1.DeviceMaintenanceAnnotation] = v1alpha1.DeviceMaintenanceResetPhase
patch.SetAnnotations(annotations)
g.Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(resource))).To(Succeed())
}).Should(Succeed())

By("Verifying the device transitions to Provisioning phase and the annotation is removed")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
_, exists := resource.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
g.Expect(exists).To(BeFalse(), "Maintenance annotation should be removed after processing")
}).Should(Succeed())
})

It("Should set Reachable=False and Ready=Unknown when the device is unreachable", func() {
By("Making the provider return a connect error")
testProvider.SetConnectError(errors.New("connection refused"))
Expand Down Expand Up @@ -682,4 +602,230 @@ var _ = Describe("Device Controller", func() {
}).Should(Succeed())
})
})

Context("When using maintenance annotations", func() {

@felix-kaestner felix-kaestner Aug 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the reason we create a new Context here instead of just adding a new It case following the pattern we already have?

var name string
var key client.ObjectKey

BeforeEach(func() {
By("Creating the endpoint credentials as a Secret")
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "test-device-",
Namespace: metav1.NamespaceDefault,
},
Data: map[string][]byte{
corev1.BasicAuthUsernameKey: []byte("user"),
corev1.BasicAuthPasswordKey: []byte("password"),
},
Type: corev1.SecretTypeBasicAuth,
}
Expect(k8sClient.Create(ctx, secret)).To(Succeed())
name = secret.Name
key = client.ObjectKey{Name: name, Namespace: metav1.NamespaceDefault}

testEvents = nil

DeferCleanup(func() {
Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, secret))).To(Succeed())

@adamtrizuljak-sap adamtrizuljak-sap Aug 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why IgnoreNotFound? The secret should still exist

})
})

It("Should transition from Running to Provisioning once the reset-phase annotation is set", func() {
By("Creating a Device")
device := &v1alpha1.Device{
ObjectMeta: metav1.ObjectMeta{
Name: key.Name,
Namespace: key.Namespace,
},
Spec: v1alpha1.DeviceSpec{
Endpoint: v1alpha1.Endpoint{
Address: "192.168.10.5:9339",
SecretRef: &v1alpha1.SecretReference{
Name: name,
},
},
Provisioning: &v1alpha1.Provisioning{
BootScript: v1alpha1.TemplateSource{
Inline: new("boot nxos.bin"),
},
Image: v1alpha1.Image{
URL: "https://best-vendor-images.to/windows98",
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
ChecksumType: v1alpha1.ChecksumTypeMD5,
},
},
},
}
Expect(k8sClient.Create(ctx, device)).To(Succeed())

By("Verifying the device transitions to Provisioning phase")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
g.Expect(resource.Status.Conditions).To(HaveLen(3))
g.Expect(resource.Status.Conditions[0].Type).To(Equal(v1alpha1.ReadyCondition))
g.Expect(resource.Status.Conditions[1].Type).To(Equal(v1alpha1.PausedCondition))
g.Expect(resource.Status.Conditions[1].Status).To(Equal(metav1.ConditionFalse))
g.Expect(resource.Status.Conditions[2].Type).To(Equal(v1alpha1.ReachableCondition))
g.Expect(resource.Status.Conditions[2].Status).To(Equal(metav1.ConditionUnknown))
}).Should(Succeed())

By("Setting the device to Running phase")
orig := device.DeepCopy()
device.Status.Phase = v1alpha1.DevicePhaseRunning
Expect(k8sClient.Status().Patch(ctx, device, client.MergeFrom(orig))).To(Succeed())

By("Verifying the device transitions to Running phase")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseRunning))
g.Expect(resource.Status.Conditions).To(HaveLen(3))
g.Expect(resource.Status.Conditions[0].Type).To(Equal(v1alpha1.ReadyCondition))
g.Expect(resource.Status.Conditions[1].Type).To(Equal(v1alpha1.PausedCondition))
g.Expect(resource.Status.Conditions[1].Status).To(Equal(metav1.ConditionFalse))
g.Expect(resource.Status.Conditions[2].Type).To(Equal(v1alpha1.ReachableCondition))
g.Expect(resource.Status.Conditions[2].Status).To(Equal(metav1.ConditionTrue))
}).Should(Succeed())

By("Adding the reset-phase annotation to the device")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
patch := resource.DeepCopy()
annotations := make(map[string]string)
annotations[v1alpha1.DeviceMaintenanceAnnotation] = v1alpha1.DeviceMaintenanceResetPhase
patch.SetAnnotations(annotations)
g.Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(resource))).To(Succeed())
}).Should(Succeed())

By("Verifying the device transitions to Provisioning phase and the annotation is removed")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
_, exists := resource.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
g.Expect(exists).To(BeFalse(), "Maintenance annotation should be removed after processing")
}).Should(Succeed())
})

It("Should skip provisioning and transition from Pending to Running when skip-provisioning annotation is set", func() {
By("Creating a Device with provisioning configured and skip-provisioning annotation")
device := &v1alpha1.Device{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "test-device-",
Namespace: metav1.NamespaceDefault,
Annotations: map[string]string{
v1alpha1.DeviceMaintenanceAnnotation: v1alpha1.DeviceMaintenanceSkipProvisioning,
},
},
Spec: v1alpha1.DeviceSpec{
Endpoint: v1alpha1.Endpoint{
Address: "192.168.10.6:9339",
SecretRef: &v1alpha1.SecretReference{
Name: name,
},
},
Provisioning: &v1alpha1.Provisioning{
BootScript: v1alpha1.TemplateSource{
Inline: new("boot nxos.bin"),
},
Image: v1alpha1.Image{
URL: "https://example.com/image.bin",
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
ChecksumType: v1alpha1.ChecksumTypeMD5,
},
},
},
}
Expect(k8sClient.Create(ctx, device)).To(Succeed())
key := client.ObjectKeyFromObject(device)

By("Verifying the device reaches Running phase with annotation removed and SkipProvisioning event emitted")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseRunning))
_, exists := resource.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
g.Expect(exists).To(BeFalse(), "skip-provisioning annotation should be removed after processing")
g.Expect(testEvents).To(ContainElement(ContainSubstring("SkipProvisioning")))
g.Expect(testEvents).NotTo(ContainElement(ContainSubstring("ProvisioningStarted")))
}).Should(Succeed())
})

It("Should close active provisioning entry and transition to Running when skip-provisioning annotation is set during Provisioning phase", func() {
By("Creating a Device with provisioning configured")
device := &v1alpha1.Device{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "test-device-",
Namespace: metav1.NamespaceDefault,
},
Spec: v1alpha1.DeviceSpec{
Endpoint: v1alpha1.Endpoint{
Address: "192.168.10.7:9339",
SecretRef: &v1alpha1.SecretReference{
Name: name,
},
},
Provisioning: &v1alpha1.Provisioning{
BootScript: v1alpha1.TemplateSource{
Inline: new("boot nxos.bin"),
},
Image: v1alpha1.Image{
URL: "https://example.com/image.bin",
Checksum: "d41d8cd98f00b204e9800998ecf8427e",
ChecksumType: v1alpha1.ChecksumTypeMD5,
},
},
},
}
Expect(k8sClient.Create(ctx, device)).To(Succeed())
key := client.ObjectKeyFromObject(device)

By("Waiting for the device to enter Provisioning phase")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseProvisioning))
}).Should(Succeed())

By("Injecting an active provisioning entry (simulating provisioning agent)")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
patch := resource.DeepCopy()
patch.Status.Provisioning = []v1alpha1.ProvisioningInfo{{
Token: "test-token",
StartTime: metav1.NewTime(time.Now().Add(-5 * time.Minute)),
}}
g.Expect(k8sClient.Status().Patch(ctx, patch, client.MergeFrom(resource))).To(Succeed())
}).Should(Succeed())

By("Adding the skip-provisioning annotation to the device")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
patch := resource.DeepCopy()
if patch.Annotations == nil {
patch.Annotations = make(map[string]string)
}
patch.Annotations[v1alpha1.DeviceMaintenanceAnnotation] = v1alpha1.DeviceMaintenanceSkipProvisioning
g.Expect(k8sClient.Patch(ctx, patch, client.MergeFrom(resource))).To(Succeed())
}).Should(Succeed())

By("Verifying the device transitions to Running phase with the provisioning entry closed")
Eventually(func(g Gomega) {
resource := &v1alpha1.Device{}
g.Expect(k8sClient.Get(ctx, key, resource)).To(Succeed())
g.Expect(resource.Status.Phase).To(Equal(v1alpha1.DevicePhaseRunning))
g.Expect(resource.Status.Provisioning).To(HaveLen(1))
g.Expect(resource.Status.Provisioning[0].EndTime.IsZero()).To(BeFalse(), "EndTime should be set on closed provisioning entry")
_, exists := resource.Annotations[v1alpha1.DeviceMaintenanceAnnotation]
g.Expect(exists).To(BeFalse(), "skip-provisioning annotation should be removed after processing")
g.Expect(testEvents).To(ContainElement(ContainSubstring("SkipProvisioning")))
}).Should(Succeed())
})
})
})
4 changes: 3 additions & 1 deletion internal/controller/core/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ var (
k8sManager ctrl.Manager
testProvider = NewProvider()
testLocker *resourcelock.ResourceLocker

// testEvents is a slice that stores events recorded during the tests. It is used to verify that the expected events are generated by the controllers.
testEvents []string
lastRebootTime = time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
)

Expand Down Expand Up @@ -100,6 +101,7 @@ var _ = BeforeSuite(func() {
go func() {
for event := range recorder.Events {
GinkgoLogr.Info("Event", "event", event)
testEvents = append(testEvents, event)
}
}()

Expand Down
Loading