diff --git a/nullplatform/agent/locals.tf b/nullplatform/agent/locals.tf index 14144903..ccd82ba0 100644 --- a/nullplatform/agent/locals.tf +++ b/nullplatform/agent/locals.tf @@ -86,6 +86,9 @@ locals { IMAGE_PULL_SECRETS = var.image_pull_secrets PRIVATE_GATEWAY_NAME = var.private_gateway_name PUBLIC_GATEWAY_NAME = var.public_gateway_name + # Name of the EKS cluster. The k8s scope needs it to look up the cluster's + # OIDC provider when it creates the IAM role for a scope. + CLUSTER_NAME = var.cluster_name } worker_cloud_config = { @@ -126,26 +129,49 @@ locals { } ] - # k8s-deployment template env vars — specific to the "containers" scope's - # worker only, regardless of what's in var.worker_orchestrated_packages. - worker_container_patch = { - target = { package = "containers" } - merge = { - spec = { - containers = [ - { - name = "worker" - env = [for k, v in local.worker_all_config : { name = k, value = v }] - } - ] + # Environment variables for the workers that run the k8s scope. + # + # The k8s scope reads its settings from env vars (DNS_TYPE, K8S_NAMESPACE, + # the template paths, CLUSTER_NAME, ...). local.worker_all_config holds all + # of them. This block turns that map into one pod patch per package listed + # in var.worker_k8s_packages, so every one of those workers boots with the + # same variables. + # + # With the default, var.worker_k8s_packages = ["containers"], the result is + # a single patch: + # + # - target: { package: containers } + # merge: + # spec: + # containers: + # - name: worker + # env: + # - { name: DNS_TYPE, value: external_dns } + # - { name: K8S_NAMESPACE, value: nullplatform } + # ... + # + # With ["containers", "scheduled-task"] you get two patches with the same + # env, one per package. Packages not in the list get none of these vars. + worker_k8s_env_patches = [ + for pkg in var.worker_k8s_packages : { + target = { package = pkg } + merge = { + spec = { + containers = [ + { + name = "worker" + env = [for k, v in local.worker_all_config : { name = k, value = v }] + } + ] + } } } - } + ] worker_defaults = { backend = "kubernetes" allowedRegistries = ["public.ecr.aws/nullplatform/*"] - patches = concat(local.worker_common_patches, [local.worker_container_patch]) + patches = concat(local.worker_common_patches, local.worker_k8s_env_patches) } worker_final = merge( diff --git a/nullplatform/agent/main.tf b/nullplatform/agent/main.tf index 6af1c998..ec4b996b 100644 --- a/nullplatform/agent/main.tf +++ b/nullplatform/agent/main.tf @@ -12,6 +12,10 @@ resource "terraform_data" "cross_variable_validation" { condition = var.cloud_provider != "aws" || var.aws_iam_role_arn != "" error_message = "aws_iam_role_arn is required when cloud_provider is 'aws'." } + precondition { + condition = var.cloud_provider != "aws" || var.cluster_name != "" || lookup(var.extra_envs, "CLUSTER_NAME", "") != "" + error_message = "cluster_name is required when cloud_provider is 'aws': the k8s scope needs it to find the EKS OIDC provider when it creates IAM roles. Set cluster_name (extra_envs.CLUSTER_NAME is still accepted for existing installations)." + } precondition { condition = var.cloud_provider != "azure" || var.azure_client_id != null error_message = "azure_client_id is required when cloud_provider is 'azure'." diff --git a/nullplatform/agent/tests/agent_values.tftest.hcl b/nullplatform/agent/tests/agent_values.tftest.hcl index ae5cc5ec..fe19d350 100644 --- a/nullplatform/agent/tests/agent_values.tftest.hcl +++ b/nullplatform/agent/tests/agent_values.tftest.hcl @@ -6,6 +6,7 @@ variables { tags_selectors = { dimension = "prod" } cloud_provider = "aws" aws_iam_role_arn = "arn:aws:iam::123456789012:role/agent" + cluster_name = "test-cluster" image_tag = "0.9.2" nullplatform_agent_helm_version = "2.37.0" agent_traffic_manager_tag = "1.8.0" @@ -433,3 +434,102 @@ run "worker_ingress_rejects_unknown_stacks" { expect_failures = [var.worker_ingress] } + +################################################################################ +# worker_k8s_packages / cluster_name +################################################################################ + +# Helper shape reused below: does the patch targeting `pkg` carry env `key`? +run "k8s_env_reaches_only_the_containers_worker_by_default" { + command = plan + + variables { + dns_type = "external_dns" + } + + assert { + condition = anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"]) + if try(p.target.package, "") == "containers" + ]) + error_message = "the containers worker must keep receiving the k8s scope env by default" + } + + assert { + condition = length([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : p + if try(p.target.package, "") != "containers" && length(try(p.merge.spec.containers[0].env, [])) > 0 + ]) == 0 + error_message = "no other package should receive the k8s scope env unless listed in worker_k8s_packages" + } +} + +run "k8s_env_reaches_every_listed_package" { + command = plan + + variables { + dns_type = "external_dns" + worker_orchestrated_packages = ["containers", "scheduled-task"] + worker_k8s_packages = ["containers", "scheduled-task"] + } + + assert { + condition = alltrue([ + for pkg in ["containers", "scheduled-task"] : + anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"]) + if try(p.target.package, "") == pkg + ]) + ]) + error_message = "every package in worker_k8s_packages must get the k8s scope env, not just containers" + } +} + +run "cluster_name_is_published_to_the_k8s_workers_when_set" { + command = plan + + variables { + cluster_name = "my-eks" + } + + assert { + condition = anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "my-eks"]) + if try(p.target.package, "") == "containers" + ]) + error_message = "cluster_name must reach the k8s worker as CLUSTER_NAME" + } +} + +run "cluster_name_is_required_on_aws" { + command = plan + + variables { + cluster_name = "" + } + + expect_failures = [ + terraform_data.cross_variable_validation, + ] +} + +run "cluster_name_can_still_arrive_through_extra_envs" { + command = plan + + variables { + cluster_name = "" + extra_envs = { CLUSTER_NAME = "legacy-eks" } + } + + assert { + condition = anytrue([ + for p in yamldecode(helm_release.agent.values[0]).worker.patches : + anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "legacy-eks"]) + if try(p.target.package, "") == "containers" + ]) + error_message = "an existing installation passing CLUSTER_NAME through extra_envs must keep working" + } +} diff --git a/nullplatform/agent/variables.tf b/nullplatform/agent/variables.tf index 21028cbf..5aa95410 100644 --- a/nullplatform/agent/variables.tf +++ b/nullplatform/agent/variables.tf @@ -84,6 +84,38 @@ variable "worker_orchestrated_packages" { default = ["containers"] } +# Packages whose worker runs the k8s scope. +# +# The k8s scope is configured through env vars (DNS_TYPE, K8S_NAMESPACE, the +# template paths, CLUSTER_NAME, ...). The module injects those vars into the +# worker pod of every package listed here. "containers" is the k8s scope +# itself. Other packages run the same k8s code from their own image with a +# few steps replaced (scheduled-task, containers-datadog); they read the same +# vars, so they belong in this list too. +# +# worker_k8s_packages = ["containers", "scheduled-task"] +# +# Packages not listed here (s3, rds, lambda, ...) do not get these vars. +variable "worker_k8s_packages" { + description = "Package slugs whose worker runs the k8s scope and must receive its env vars (DNS_TYPE, K8S_NAMESPACE, template paths, CLUSTER_NAME, extra_envs). Add every package that runs the k8s scope code, e.g. [\"containers\", \"scheduled-task\"]." + type = list(string) + default = ["containers"] +} + +# Name of the Kubernetes cluster the scopes are deployed to. +# +# The k8s scope uses it to find the cluster's OIDC provider when it creates +# the IAM role for a scope. It reaches the workers as the CLUSTER_NAME env var. +# Required when cloud_provider is "aws" (enforced by a precondition in main.tf, +# so a missing value fails at plan time instead of inside create-scope). +# +# cluster_name = module.eks.eks_cluster_name +variable "cluster_name" { + description = "Name of the Kubernetes cluster the scopes run in. Sent to the k8s workers as CLUSTER_NAME; the k8s scope uses it to find the EKS OIDC provider when creating IAM roles. Required when cloud_provider is 'aws'." + type = string + default = "" +} + variable "worker_memory_limit" { description = "Memory limit for a worker-orchestrated package's pod (packages in var.worker_orchestrated_packages). The chart's own default is small enough to OOM mid-tofu-apply for packages that run real IaC tooling." type = string