From ea2656398ef6642b83295ce73caedc6044b79d57 Mon Sep 17 00:00:00 2001 From: Matt Pryor Date: Sun, 30 Aug 2026 16:05:38 +0100 Subject: [PATCH] Clean up orphaned sveltos-agent version ConfigMaps removeSveltosAgentFromManagementCluster tears down every resource in the sveltos-agent manifest for a cluster that's gone, but never touched the sa-* version ConfigMap since that gets written by the running agent itself, not deployed as part of the manifest. Left unowned and never cleaned up, these accumulate without bound - one NKS cell had ~2000 of them from nothing but a canary probe's create/delete cycle. Delete it last, after every manifest resource (including the deployment) already has a delete issued, and propagate the error instead of swallowing it like the rest of that function does: cleanClusterStaleResources turns a failure here into a requeue, so this gets retried through both the reconciler and the periodic sweep instead of just giving up. --- config/rbac/role.yaml | 8 ++++++ controllers/classifier_controller.go | 2 +- controllers/classifier_deployer.go | 10 +++++++ controllers/classifier_deployer_test.go | 36 +++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- manifest/manifest.yaml | 8 ++++++ 7 files changed, 66 insertions(+), 4 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index bc2ef54..7bf52b8 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -8,6 +8,14 @@ rules: - "" resources: - configmaps + verbs: + - delete + - get + - list + - watch +- apiGroups: + - "" + resources: - secrets verbs: - get diff --git a/controllers/classifier_controller.go b/controllers/classifier_controller.go index 5dbcfcf..12566c8 100644 --- a/controllers/classifier_controller.go +++ b/controllers/classifier_controller.go @@ -131,7 +131,7 @@ type ClassifierReconciler struct { //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;watch;list;update //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters/status,verbs=get;watch;list //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch -//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch +//+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;delete //+kubebuilder:rbac:groups="apps",resources=deployments,verbs=get;list;watch func (r *ClassifierReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/classifier_deployer.go b/controllers/classifier_deployer.go index 860e127..56b3742 100644 --- a/controllers/classifier_deployer.go +++ b/controllers/classifier_deployer.go @@ -56,6 +56,7 @@ import ( logs "github.com/projectsveltos/libsveltos/lib/logsettings" "github.com/projectsveltos/libsveltos/lib/patcher" "github.com/projectsveltos/libsveltos/lib/pullmode" + "github.com/projectsveltos/libsveltos/lib/sveltos_upgrade" ) type getCurrentHash func(classifier *libsveltosv1beta1.Classifier) []byte @@ -2072,6 +2073,15 @@ func removeSveltosAgentFromManagementCluster(ctx context.Context, manager.RemoveSveltosAgentDeploymentName(clusterNamespace, clusterName, clusterType) } + // Propagate the error here (unlike the deletes in the loop above): cleanClusterStaleResources + // turns this into a requeue, so it gets retried via the reconciler and the periodic sweep + // instead of leaving the version ConfigMap behind for good. + if err := sveltos_upgrade.DeleteSveltosAgentVersion(ctx, getManagementClusterClient(), getSveltosNamespace(), + clusterNamespace, clusterName, clusterType, true, logger); err != nil { + logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to delete sveltos-agent version configMap: %v", err)) + return err + } + return nil } diff --git a/controllers/classifier_deployer_test.go b/controllers/classifier_deployer_test.go index 1f3ebfd..bed2883 100644 --- a/controllers/classifier_deployer_test.go +++ b/controllers/classifier_deployer_test.go @@ -44,6 +44,7 @@ import ( libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/deployer" fakedeployer "github.com/projectsveltos/libsveltos/lib/deployer/fake" + "github.com/projectsveltos/libsveltos/lib/sveltos_upgrade" ) const ( @@ -592,6 +593,17 @@ var _ = Describe("Classifier Deployer", func() { clusterName := randomString() clusterType := libsveltosv1beta1.ClusterTypeSveltos + // The version ConfigMap is created in clusterNamespace itself (matching a real CAPI/Sveltos + // cluster's namespace), so - unlike the rest of this test, which never touches clusterNamespace + // as a real object - it needs to actually exist for that Create to pass namespace admission. + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: clusterNamespace, + }, + } + Expect(testEnv.Create(context.TODO(), ns)).To(Succeed()) + Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed()) + _, err := keymanager.GetKeyManagerInstance(context.TODO(), testEnv.Client) Expect(err).To(BeNil()) @@ -623,6 +635,23 @@ var _ = Describe("Classifier Deployer", func() { return false }, timeout, pollingInterval).Should(BeTrue()) + Expect(sveltos_upgrade.StoreSveltosAgentVersion(context.TODO(), testEnv.Client, sveltosNamespace, "v1.0.0", + clusterNamespace, clusterName, clusterType, true, logger)).To(Succeed()) + + var versionConfigMap corev1.ConfigMap + Eventually(func() bool { + versionConfigMaps := &corev1.ConfigMapList{} + err := testEnv.List(context.TODO(), versionConfigMaps, client.InNamespace(clusterNamespace), client.MatchingLabels{ + sveltos_upgrade.ClusterNameLabel: clusterName, + sveltos_upgrade.ClusterTypeLabel: strings.ToLower(string(clusterType)), + }) + if err != nil || len(versionConfigMaps.Items) != 1 { + return false + } + versionConfigMap = versionConfigMaps.Items[0] + return true + }, timeout, pollingInterval).Should(BeTrue()) + Expect(controllers.RemoveSveltosAgentFromManagementCluster(context.TODO(), clusterNamespace, clusterName, clusterType, logger)).To(Succeed()) @@ -641,6 +670,13 @@ var _ = Describe("Classifier Deployer", func() { } return true }, timeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + err := testEnv.Get(context.TODO(), + types.NamespacedName{Namespace: versionConfigMap.Namespace, Name: versionConfigMap.Name}, + &corev1.ConfigMap{}) + return apierrors.IsNotFound(err) + }, timeout, pollingInterval).Should(BeTrue()) }) It("getSveltosAgentPatches reads post render patches from ConfigMap", func() { diff --git a/go.mod b/go.mod index 804d6ea..b1ef785 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/onsi/ginkgo/v2 v2.32.1 github.com/onsi/gomega v1.42.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.14.0 + github.com/projectsveltos/libsveltos v1.14.1-0.20260830170459-49d8e4e94b6e github.com/prometheus/client_golang v1.24.1 github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.2 diff --git a/go.sum b/go.sum index 921a814..81c39c9 100644 --- a/go.sum +++ b/go.sum @@ -206,8 +206,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/projectsveltos/libsveltos v1.14.0 h1:vw+kbGfsMcKk69AdQsjpdhlZK0LI/07Y+292noB9F+8= -github.com/projectsveltos/libsveltos v1.14.0/go.mod h1:U6iGj5KoC/PcTD2vh3XU6gy7g11suThT6sZSEpmLEkU= +github.com/projectsveltos/libsveltos v1.14.1-0.20260830170459-49d8e4e94b6e h1:jIOPS81hUuFNwGCTgGV/M4eFbsVCd1vpByYgWYMyngg= +github.com/projectsveltos/libsveltos v1.14.1-0.20260830170459-49d8e4e94b6e/go.mod h1:U6iGj5KoC/PcTD2vh3XU6gy7g11suThT6sZSEpmLEkU= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 0ee823a..223b717 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -23,6 +23,14 @@ rules: - "" resources: - configmaps + verbs: + - delete + - get + - list + - watch +- apiGroups: + - "" + resources: - secrets verbs: - get