From 751b5312fd8300910d6d26c1ee8d98ef4f8c13e8 Mon Sep 17 00:00:00 2001 From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:58:33 -0300 Subject: [PATCH 1/4] feat(agent): publish the k8s scope env to every k8s worker and take cluster_name as input The deploy/DNS env used to be hardcoded into a patch for the containers package only. Any other package running the k8s scope code under an overlay (scheduled task, datadog) needed a hand-written patch in the root module, and CLUSTER_NAME could only reach the worker via extra_envs. - worker_k8s_packages (default ["containers"]): one env patch per listed package, replacing worker_container_patch. - cluster_name: published as CLUSTER_NAME to those workers when set; omitted when empty so it cannot shadow a value passed via extra_envs. Default output is unchanged. Four new tests. --- nullplatform/agent/locals.tf | 64 ++++++++------- .../agent/tests/agent_values.tftest.hcl | 81 +++++++++++++++++++ nullplatform/agent/variables.tf | 19 +++++ 3 files changed, 137 insertions(+), 27 deletions(-) diff --git a/nullplatform/agent/locals.tf b/nullplatform/agent/locals.tf index 14144903..b5c492e3 100644 --- a/nullplatform/agent/locals.tf +++ b/nullplatform/agent/locals.tf @@ -74,19 +74,25 @@ locals { } worker_templates = local.worker_ingress_templates[var.worker_ingress] - worker_default_env = { - DNS_TYPE = var.dns_type - DOMAIN = var.domain - USE_ACCOUNT_SLUG = var.use_account_slug - K8S_NAMESPACE = var.namespace - SERVICE_TEMPLATE = var.service_template != "" ? var.service_template : local.worker_templates.SERVICE_TEMPLATE - INITIAL_INGRESS_PATH = var.initial_ingress_path != "" ? var.initial_ingress_path : local.worker_templates.INITIAL_INGRESS_PATH - BLUE_GREEN_INGRESS_PATH = var.blue_green_ingress_path != "" ? var.blue_green_ingress_path : local.worker_templates.BLUE_GREEN_INGRESS_PATH - TRAFFIC_CONTAINER_IMAGE = "${var.agent_traffic_manager_repository}:${var.agent_traffic_manager_tag}" - IMAGE_PULL_SECRETS = var.image_pull_secrets - PRIVATE_GATEWAY_NAME = var.private_gateway_name - PUBLIC_GATEWAY_NAME = var.public_gateway_name - } + worker_default_env = merge( + { + DNS_TYPE = var.dns_type + DOMAIN = var.domain + USE_ACCOUNT_SLUG = var.use_account_slug + K8S_NAMESPACE = var.namespace + SERVICE_TEMPLATE = var.service_template != "" ? var.service_template : local.worker_templates.SERVICE_TEMPLATE + INITIAL_INGRESS_PATH = var.initial_ingress_path != "" ? var.initial_ingress_path : local.worker_templates.INITIAL_INGRESS_PATH + BLUE_GREEN_INGRESS_PATH = var.blue_green_ingress_path != "" ? var.blue_green_ingress_path : local.worker_templates.BLUE_GREEN_INGRESS_PATH + TRAFFIC_CONTAINER_IMAGE = "${var.agent_traffic_manager_repository}:${var.agent_traffic_manager_tag}" + IMAGE_PULL_SECRETS = var.image_pull_secrets + PRIVATE_GATEWAY_NAME = var.private_gateway_name + PUBLIC_GATEWAY_NAME = var.public_gateway_name + }, + # scope/iam/create_role reads the cluster name from the env to find the EKS + # OIDC provider; only sent when set so an empty value cannot shadow one + # still passed through extra_envs. + var.cluster_name != "" ? { CLUSTER_NAME = var.cluster_name } : {}, + ) worker_cloud_config = { azure = { @@ -126,26 +132,30 @@ locals { } ] - # k8s-deployment template env vars — specific to the "containers" scope's - # worker only, regardless of what's in var.worker_orchestrated_packages. - worker_container_patch = { - target = { package = "containers" } - merge = { - spec = { - containers = [ - { - name = "worker" - env = [for k, v in local.worker_all_config : { name = k, value = v }] - } - ] + # k8s scope env (deploy/DNS templates, namespace, cluster) — one patch per + # package in var.worker_k8s_packages: the "containers" worker by default, + # plus any package that runs the k8s scope code under an overlay (scheduled + # task, datadog, ...). Packages outside that list get none of it. + worker_k8s_env_patches = [ + for pkg in var.worker_k8s_packages : { + target = { package = pkg } + merge = { + spec = { + containers = [ + { + name = "worker" + env = [for k, v in local.worker_all_config : { name = k, value = v }] + } + ] + } } } - } + ] worker_defaults = { backend = "kubernetes" allowedRegistries = ["public.ecr.aws/nullplatform/*"] - patches = concat(local.worker_common_patches, [local.worker_container_patch]) + patches = concat(local.worker_common_patches, local.worker_k8s_env_patches) } worker_final = merge( diff --git a/nullplatform/agent/tests/agent_values.tftest.hcl b/nullplatform/agent/tests/agent_values.tftest.hcl index ae5cc5ec..6b9340bf 100644 --- a/nullplatform/agent/tests/agent_values.tftest.hcl +++ b/nullplatform/agent/tests/agent_values.tftest.hcl @@ -433,3 +433,84 @@ run "worker_ingress_rejects_unknown_stacks" { expect_failures = [var.worker_ingress] } + +################################################################################ +# worker_k8s_packages / cluster_name +################################################################################ + +# Helper shape reused below: does the patch targeting `pkg` carry env `key`? +run "k8s_env_reaches_only_the_containers_worker_by_default" { + command = plan + + variables { + dns_type = "external_dns" + } + + assert { + condition = anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"]) + if try(p.target.package, "") == "containers" + ]) + error_message = "the containers worker must keep receiving the k8s scope env by default" + } + + assert { + condition = length([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : p + if try(p.target.package, "") != "containers" && length(try(p.merge.spec.containers[0].env, [])) > 0 + ]) == 0 + error_message = "no other package should receive the k8s scope env unless listed in worker_k8s_packages" + } +} + +run "k8s_env_reaches_every_listed_package" { + command = plan + + variables { + dns_type = "external_dns" + worker_orchestrated_packages = ["containers", "scheduled-task"] + worker_k8s_packages = ["containers", "scheduled-task"] + } + + assert { + condition = alltrue([ + for pkg in ["containers", "scheduled-task"] : + anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"]) + if try(p.target.package, "") == pkg + ]) + ]) + error_message = "every package in worker_k8s_packages must get the k8s scope env, not just containers" + } +} + +run "cluster_name_is_published_to_the_k8s_workers_when_set" { + command = plan + + variables { + cluster_name = "my-eks" + } + + assert { + condition = anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "my-eks"]) + if try(p.target.package, "") == "containers" + ]) + error_message = "cluster_name must reach the k8s worker as CLUSTER_NAME" + } +} + +run "cluster_name_is_omitted_when_empty" { + command = plan + + assert { + condition = !anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME"]) + ]) + error_message = "an unset cluster_name must not publish an empty CLUSTER_NAME that would shadow extra_envs" + } +} diff --git a/nullplatform/agent/variables.tf b/nullplatform/agent/variables.tf index 21028cbf..0116f9a4 100644 --- a/nullplatform/agent/variables.tf +++ b/nullplatform/agent/variables.tf @@ -84,6 +84,25 @@ variable "worker_orchestrated_packages" { default = ["containers"] } +# Which worker-orchestrated packages run the k8s scope code and therefore need +# its env (deploy/DNS templates, namespace, cluster name). "containers" is the +# scope itself; overlays that run the same code from their own image, such as +# scheduled task or datadog, need the very same variables and belong here too. +variable "worker_k8s_packages" { + description = "Package slugs whose worker runs the k8s scope code and receives its env (DNS_TYPE, K8S_NAMESPACE, the template paths, TRAFFIC_CONTAINER_IMAGE, CLUSTER_NAME, extra_envs). Defaults to the containers scope; add overlays such as scheduled-task so they stop needing a hand-written patch." + type = list(string) + default = ["containers"] +} + +# EKS cluster name for the k8s scope's create_role (it resolves the OIDC +# provider from it). Previously it could only reach the worker through +# extra_envs. +variable "cluster_name" { + description = "Kubernetes cluster name the k8s scope's create_role uses to find the EKS OIDC provider. Published to the k8s workers as CLUSTER_NAME when set." + type = string + default = "" +} + variable "worker_memory_limit" { description = "Memory limit for a worker-orchestrated package's pod (packages in var.worker_orchestrated_packages). The chart's own default is small enough to OOM mid-tofu-apply for packages that run real IaC tooling." type = string From e28a56526e18ddc8b6d0fb87b5f61237cbe6bfce Mon Sep 17 00:00:00 2001 From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:22:56 -0300 Subject: [PATCH 2/4] refactor(agent): publish CLUSTER_NAME unconditionally like the rest of the k8s env extra_envs is merged last, so a value passed there always wins; and create_role reads $CLUSTER_NAME directly, for which empty and unset are the same. The conditional bought nothing. --- nullplatform/agent/locals.tf | 33 ++++++++----------- .../agent/tests/agent_values.tftest.hcl | 9 ++--- nullplatform/agent/variables.tf | 2 +- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/nullplatform/agent/locals.tf b/nullplatform/agent/locals.tf index b5c492e3..2b96c09a 100644 --- a/nullplatform/agent/locals.tf +++ b/nullplatform/agent/locals.tf @@ -74,25 +74,20 @@ locals { } worker_templates = local.worker_ingress_templates[var.worker_ingress] - worker_default_env = merge( - { - DNS_TYPE = var.dns_type - DOMAIN = var.domain - USE_ACCOUNT_SLUG = var.use_account_slug - K8S_NAMESPACE = var.namespace - SERVICE_TEMPLATE = var.service_template != "" ? var.service_template : local.worker_templates.SERVICE_TEMPLATE - INITIAL_INGRESS_PATH = var.initial_ingress_path != "" ? var.initial_ingress_path : local.worker_templates.INITIAL_INGRESS_PATH - BLUE_GREEN_INGRESS_PATH = var.blue_green_ingress_path != "" ? var.blue_green_ingress_path : local.worker_templates.BLUE_GREEN_INGRESS_PATH - TRAFFIC_CONTAINER_IMAGE = "${var.agent_traffic_manager_repository}:${var.agent_traffic_manager_tag}" - IMAGE_PULL_SECRETS = var.image_pull_secrets - PRIVATE_GATEWAY_NAME = var.private_gateway_name - PUBLIC_GATEWAY_NAME = var.public_gateway_name - }, - # scope/iam/create_role reads the cluster name from the env to find the EKS - # OIDC provider; only sent when set so an empty value cannot shadow one - # still passed through extra_envs. - var.cluster_name != "" ? { CLUSTER_NAME = var.cluster_name } : {}, - ) + worker_default_env = { + DNS_TYPE = var.dns_type + DOMAIN = var.domain + USE_ACCOUNT_SLUG = var.use_account_slug + K8S_NAMESPACE = var.namespace + SERVICE_TEMPLATE = var.service_template != "" ? var.service_template : local.worker_templates.SERVICE_TEMPLATE + INITIAL_INGRESS_PATH = var.initial_ingress_path != "" ? var.initial_ingress_path : local.worker_templates.INITIAL_INGRESS_PATH + BLUE_GREEN_INGRESS_PATH = var.blue_green_ingress_path != "" ? var.blue_green_ingress_path : local.worker_templates.BLUE_GREEN_INGRESS_PATH + TRAFFIC_CONTAINER_IMAGE = "${var.agent_traffic_manager_repository}:${var.agent_traffic_manager_tag}" + IMAGE_PULL_SECRETS = var.image_pull_secrets + PRIVATE_GATEWAY_NAME = var.private_gateway_name + PUBLIC_GATEWAY_NAME = var.public_gateway_name + CLUSTER_NAME = var.cluster_name + } worker_cloud_config = { azure = { diff --git a/nullplatform/agent/tests/agent_values.tftest.hcl b/nullplatform/agent/tests/agent_values.tftest.hcl index 6b9340bf..39dbd70e 100644 --- a/nullplatform/agent/tests/agent_values.tftest.hcl +++ b/nullplatform/agent/tests/agent_values.tftest.hcl @@ -503,14 +503,15 @@ run "cluster_name_is_published_to_the_k8s_workers_when_set" { } } -run "cluster_name_is_omitted_when_empty" { +run "cluster_name_defaults_to_empty_like_the_other_scope_env" { command = plan assert { - condition = !anytrue([ + condition = anytrue([ for p in yamldecode(helm_release.agent.values[0]).worker.patches : - anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME"]) + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == ""]) + if try(p.target.package, "") == "containers" ]) - error_message = "an unset cluster_name must not publish an empty CLUSTER_NAME that would shadow extra_envs" + error_message = "an unset cluster_name renders as an empty CLUSTER_NAME, the same as every other unset scope variable" } } diff --git a/nullplatform/agent/variables.tf b/nullplatform/agent/variables.tf index 0116f9a4..64d68eb0 100644 --- a/nullplatform/agent/variables.tf +++ b/nullplatform/agent/variables.tf @@ -98,7 +98,7 @@ variable "worker_k8s_packages" { # provider from it). Previously it could only reach the worker through # extra_envs. variable "cluster_name" { - description = "Kubernetes cluster name the k8s scope's create_role uses to find the EKS OIDC provider. Published to the k8s workers as CLUSTER_NAME when set." + description = "Kubernetes cluster name the k8s scope's create_role uses to find the EKS OIDC provider. Published to the k8s workers as CLUSTER_NAME." type = string default = "" } From 634df1b5f7564a232ec335f074cd61333cf1dde2 Mon Sep 17 00:00:00 2001 From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:26:24 -0300 Subject: [PATCH 3/4] docs(agent): explain worker_k8s_packages and cluster_name for first-time readers --- nullplatform/agent/locals.tf | 31 ++++++++++++++++++++++++++----- nullplatform/agent/variables.tf | 29 ++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/nullplatform/agent/locals.tf b/nullplatform/agent/locals.tf index 2b96c09a..ccd82ba0 100644 --- a/nullplatform/agent/locals.tf +++ b/nullplatform/agent/locals.tf @@ -86,7 +86,9 @@ locals { IMAGE_PULL_SECRETS = var.image_pull_secrets PRIVATE_GATEWAY_NAME = var.private_gateway_name PUBLIC_GATEWAY_NAME = var.public_gateway_name - CLUSTER_NAME = var.cluster_name + # Name of the EKS cluster. The k8s scope needs it to look up the cluster's + # OIDC provider when it creates the IAM role for a scope. + CLUSTER_NAME = var.cluster_name } worker_cloud_config = { @@ -127,10 +129,29 @@ locals { } ] - # k8s scope env (deploy/DNS templates, namespace, cluster) — one patch per - # package in var.worker_k8s_packages: the "containers" worker by default, - # plus any package that runs the k8s scope code under an overlay (scheduled - # task, datadog, ...). Packages outside that list get none of it. + # Environment variables for the workers that run the k8s scope. + # + # The k8s scope reads its settings from env vars (DNS_TYPE, K8S_NAMESPACE, + # the template paths, CLUSTER_NAME, ...). local.worker_all_config holds all + # of them. This block turns that map into one pod patch per package listed + # in var.worker_k8s_packages, so every one of those workers boots with the + # same variables. + # + # With the default, var.worker_k8s_packages = ["containers"], the result is + # a single patch: + # + # - target: { package: containers } + # merge: + # spec: + # containers: + # - name: worker + # env: + # - { name: DNS_TYPE, value: external_dns } + # - { name: K8S_NAMESPACE, value: nullplatform } + # ... + # + # With ["containers", "scheduled-task"] you get two patches with the same + # env, one per package. Packages not in the list get none of these vars. worker_k8s_env_patches = [ for pkg in var.worker_k8s_packages : { target = { package = pkg } diff --git a/nullplatform/agent/variables.tf b/nullplatform/agent/variables.tf index 64d68eb0..9f8a9973 100644 --- a/nullplatform/agent/variables.tf +++ b/nullplatform/agent/variables.tf @@ -84,21 +84,32 @@ variable "worker_orchestrated_packages" { default = ["containers"] } -# Which worker-orchestrated packages run the k8s scope code and therefore need -# its env (deploy/DNS templates, namespace, cluster name). "containers" is the -# scope itself; overlays that run the same code from their own image, such as -# scheduled task or datadog, need the very same variables and belong here too. +# Packages whose worker runs the k8s scope. +# +# The k8s scope is configured through env vars (DNS_TYPE, K8S_NAMESPACE, the +# template paths, CLUSTER_NAME, ...). The module injects those vars into the +# worker pod of every package listed here. "containers" is the k8s scope +# itself. Other packages run the same k8s code from their own image with a +# few steps replaced (scheduled-task, containers-datadog); they read the same +# vars, so they belong in this list too. +# +# worker_k8s_packages = ["containers", "scheduled-task"] +# +# Packages not listed here (s3, rds, lambda, ...) do not get these vars. variable "worker_k8s_packages" { - description = "Package slugs whose worker runs the k8s scope code and receives its env (DNS_TYPE, K8S_NAMESPACE, the template paths, TRAFFIC_CONTAINER_IMAGE, CLUSTER_NAME, extra_envs). Defaults to the containers scope; add overlays such as scheduled-task so they stop needing a hand-written patch." + description = "Package slugs whose worker runs the k8s scope and must receive its env vars (DNS_TYPE, K8S_NAMESPACE, template paths, CLUSTER_NAME, extra_envs). Add every package that runs the k8s scope code, e.g. [\"containers\", \"scheduled-task\"]." type = list(string) default = ["containers"] } -# EKS cluster name for the k8s scope's create_role (it resolves the OIDC -# provider from it). Previously it could only reach the worker through -# extra_envs. +# Name of the Kubernetes cluster the scopes are deployed to. +# +# The k8s scope uses it to find the cluster's OIDC provider when it creates +# the IAM role for a scope. It reaches the workers as the CLUSTER_NAME env var. +# +# cluster_name = module.eks.eks_cluster_name variable "cluster_name" { - description = "Kubernetes cluster name the k8s scope's create_role uses to find the EKS OIDC provider. Published to the k8s workers as CLUSTER_NAME." + description = "Name of the Kubernetes cluster the scopes run in. Sent to the k8s workers as CLUSTER_NAME; the k8s scope uses it to find the EKS OIDC provider when creating IAM roles." type = string default = "" } From d471a12365a7ab4bf32fbd544c1252e91b18259a Mon Sep 17 00:00:00 2001 From: Agustin Celentano <12614595+agustincelentano@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:10:38 -0300 Subject: [PATCH 4/4] fix(agent): require cluster_name on aws instead of defaulting it to empty The k8s scope needs it to resolve the EKS OIDC provider; an empty value only surfaced inside create-scope as an AWS error. Same precondition pattern as aws_iam_role_arn; extra_envs.CLUSTER_NAME is still accepted so existing installations keep working. --- nullplatform/agent/main.tf | 4 ++++ .../agent/tests/agent_values.tftest.hcl | 24 ++++++++++++++++--- nullplatform/agent/variables.tf | 4 +++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/nullplatform/agent/main.tf b/nullplatform/agent/main.tf index 6af1c998..ec4b996b 100644 --- a/nullplatform/agent/main.tf +++ b/nullplatform/agent/main.tf @@ -12,6 +12,10 @@ resource "terraform_data" "cross_variable_validation" { condition = var.cloud_provider != "aws" || var.aws_iam_role_arn != "" error_message = "aws_iam_role_arn is required when cloud_provider is 'aws'." } + precondition { + condition = var.cloud_provider != "aws" || var.cluster_name != "" || lookup(var.extra_envs, "CLUSTER_NAME", "") != "" + error_message = "cluster_name is required when cloud_provider is 'aws': the k8s scope needs it to find the EKS OIDC provider when it creates IAM roles. Set cluster_name (extra_envs.CLUSTER_NAME is still accepted for existing installations)." + } precondition { condition = var.cloud_provider != "azure" || var.azure_client_id != null error_message = "azure_client_id is required when cloud_provider is 'azure'." diff --git a/nullplatform/agent/tests/agent_values.tftest.hcl b/nullplatform/agent/tests/agent_values.tftest.hcl index 39dbd70e..fe19d350 100644 --- a/nullplatform/agent/tests/agent_values.tftest.hcl +++ b/nullplatform/agent/tests/agent_values.tftest.hcl @@ -6,6 +6,7 @@ variables { tags_selectors = { dimension = "prod" } cloud_provider = "aws" aws_iam_role_arn = "arn:aws:iam::123456789012:role/agent" + cluster_name = "test-cluster" image_tag = "0.9.2" nullplatform_agent_helm_version = "2.37.0" agent_traffic_manager_tag = "1.8.0" @@ -503,15 +504,32 @@ run "cluster_name_is_published_to_the_k8s_workers_when_set" { } } -run "cluster_name_defaults_to_empty_like_the_other_scope_env" { +run "cluster_name_is_required_on_aws" { command = plan + variables { + cluster_name = "" + } + + expect_failures = [ + terraform_data.cross_variable_validation, + ] +} + +run "cluster_name_can_still_arrive_through_extra_envs" { + command = plan + + variables { + cluster_name = "" + extra_envs = { CLUSTER_NAME = "legacy-eks" } + } + assert { condition = anytrue([ for p in yamldecode(helm_release.agent.values[0]).worker.patches : - anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == ""]) + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "legacy-eks"]) if try(p.target.package, "") == "containers" ]) - error_message = "an unset cluster_name renders as an empty CLUSTER_NAME, the same as every other unset scope variable" + error_message = "an existing installation passing CLUSTER_NAME through extra_envs must keep working" } } diff --git a/nullplatform/agent/variables.tf b/nullplatform/agent/variables.tf index 9f8a9973..5aa95410 100644 --- a/nullplatform/agent/variables.tf +++ b/nullplatform/agent/variables.tf @@ -106,10 +106,12 @@ variable "worker_k8s_packages" { # # The k8s scope uses it to find the cluster's OIDC provider when it creates # the IAM role for a scope. It reaches the workers as the CLUSTER_NAME env var. +# Required when cloud_provider is "aws" (enforced by a precondition in main.tf, +# so a missing value fails at plan time instead of inside create-scope). # # cluster_name = module.eks.eks_cluster_name variable "cluster_name" { - description = "Name of the Kubernetes cluster the scopes run in. Sent to the k8s workers as CLUSTER_NAME; the k8s scope uses it to find the EKS OIDC provider when creating IAM roles." + description = "Name of the Kubernetes cluster the scopes run in. Sent to the k8s workers as CLUSTER_NAME; the k8s scope uses it to find the EKS OIDC provider when creating IAM roles. Required when cloud_provider is 'aws'." type = string default = "" }