Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 40 additions & 14 deletions nullplatform/agent/locals.tf
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,9 @@ locals {
IMAGE_PULL_SECRETS = var.image_pull_secrets
PRIVATE_GATEWAY_NAME = var.private_gateway_name
PUBLIC_GATEWAY_NAME = var.public_gateway_name
# Name of the EKS cluster. The k8s scope needs it to look up the cluster's
# OIDC provider when it creates the IAM role for a scope.
CLUSTER_NAME = var.cluster_name
}

worker_cloud_config = {
Expand Down Expand Up @@ -126,26 +129,49 @@ locals {
}
]

# k8s-deployment template env vars — specific to the "containers" scope's
# worker only, regardless of what's in var.worker_orchestrated_packages.
worker_container_patch = {
target = { package = "containers" }
merge = {
spec = {
containers = [
{
name = "worker"
env = [for k, v in local.worker_all_config : { name = k, value = v }]
}
]
# Environment variables for the workers that run the k8s scope.
#
# The k8s scope reads its settings from env vars (DNS_TYPE, K8S_NAMESPACE,
# the template paths, CLUSTER_NAME, ...). local.worker_all_config holds all
# of them. This block turns that map into one pod patch per package listed
# in var.worker_k8s_packages, so every one of those workers boots with the
# same variables.
#
# With the default, var.worker_k8s_packages = ["containers"], the result is
# a single patch:
#
# - target: { package: containers }
# merge:
# spec:
# containers:
# - name: worker
# env:
# - { name: DNS_TYPE, value: external_dns }
# - { name: K8S_NAMESPACE, value: nullplatform }
# ...
#
# With ["containers", "scheduled-task"] you get two patches with the same
# env, one per package. Packages not in the list get none of these vars.
worker_k8s_env_patches = [
for pkg in var.worker_k8s_packages : {
target = { package = pkg }
merge = {
spec = {
containers = [
{
name = "worker"
env = [for k, v in local.worker_all_config : { name = k, value = v }]
}
]
}
}
}
}
]

worker_defaults = {
backend = "kubernetes"
allowedRegistries = ["public.ecr.aws/nullplatform/*"]
patches = concat(local.worker_common_patches, [local.worker_container_patch])
patches = concat(local.worker_common_patches, local.worker_k8s_env_patches)
}

worker_final = merge(
Expand Down
4 changes: 4 additions & 0 deletions nullplatform/agent/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ resource "terraform_data" "cross_variable_validation" {
condition = var.cloud_provider != "aws" || var.aws_iam_role_arn != ""
error_message = "aws_iam_role_arn is required when cloud_provider is 'aws'."
}
precondition {
condition = var.cloud_provider != "aws" || var.cluster_name != "" || lookup(var.extra_envs, "CLUSTER_NAME", "") != ""
error_message = "cluster_name is required when cloud_provider is 'aws': the k8s scope needs it to find the EKS OIDC provider when it creates IAM roles. Set cluster_name (extra_envs.CLUSTER_NAME is still accepted for existing installations)."
}
precondition {
condition = var.cloud_provider != "azure" || var.azure_client_id != null
error_message = "azure_client_id is required when cloud_provider is 'azure'."
Expand Down
100 changes: 100 additions & 0 deletions nullplatform/agent/tests/agent_values.tftest.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ variables {
tags_selectors = { dimension = "prod" }
cloud_provider = "aws"
aws_iam_role_arn = "arn:aws:iam::123456789012:role/agent"
cluster_name = "test-cluster"
image_tag = "0.9.2"
nullplatform_agent_helm_version = "2.37.0"
agent_traffic_manager_tag = "1.8.0"
Expand Down Expand Up @@ -433,3 +434,102 @@ run "worker_ingress_rejects_unknown_stacks" {

expect_failures = [var.worker_ingress]
}

################################################################################
# worker_k8s_packages / cluster_name
################################################################################

# Helper shape reused below: does the patch targeting `pkg` carry env `key`?
run "k8s_env_reaches_only_the_containers_worker_by_default" {
command = plan

variables {
dns_type = "external_dns"
}

assert {
condition = anytrue([
for p in yamldecode(helm_release.agent.values[0]).worker.patches :
anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"])
if try(p.target.package, "") == "containers"
])
error_message = "the containers worker must keep receiving the k8s scope env by default"
}

assert {
condition = length([
for p in yamldecode(helm_release.agent.values[0]).worker.patches : p
if try(p.target.package, "") != "containers" && length(try(p.merge.spec.containers[0].env, [])) > 0
]) == 0
error_message = "no other package should receive the k8s scope env unless listed in worker_k8s_packages"
}
}

run "k8s_env_reaches_every_listed_package" {
command = plan

variables {
dns_type = "external_dns"
worker_orchestrated_packages = ["containers", "scheduled-task"]
worker_k8s_packages = ["containers", "scheduled-task"]
}

assert {
condition = alltrue([
for pkg in ["containers", "scheduled-task"] :
anytrue([
for p in yamldecode(helm_release.agent.values[0]).worker.patches :
anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "DNS_TYPE" && e.value == "external_dns"])
if try(p.target.package, "") == pkg
])
])
error_message = "every package in worker_k8s_packages must get the k8s scope env, not just containers"
}
}

run "cluster_name_is_published_to_the_k8s_workers_when_set" {
command = plan

variables {
cluster_name = "my-eks"
}

assert {
condition = anytrue([
for p in yamldecode(helm_release.agent.values[0]).worker.patches :
anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "my-eks"])
if try(p.target.package, "") == "containers"
])
error_message = "cluster_name must reach the k8s worker as CLUSTER_NAME"
}
}

run "cluster_name_is_required_on_aws" {
command = plan

variables {
cluster_name = ""
}

expect_failures = [
terraform_data.cross_variable_validation,
]
}

run "cluster_name_can_still_arrive_through_extra_envs" {
command = plan

variables {
cluster_name = ""
extra_envs = { CLUSTER_NAME = "legacy-eks" }
}

assert {
condition = anytrue([
for p in yamldecode(helm_release.agent.values[0]).worker.patches :
anytrue([for e in try(p.merge.spec.containers[0].env, []) : e.name == "CLUSTER_NAME" && e.value == "legacy-eks"])
if try(p.target.package, "") == "containers"
])
error_message = "an existing installation passing CLUSTER_NAME through extra_envs must keep working"
}
}
32 changes: 32 additions & 0 deletions nullplatform/agent/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,38 @@ variable "worker_orchestrated_packages" {
default = ["containers"]
}

# Packages whose worker runs the k8s scope.
#
# The k8s scope is configured through env vars (DNS_TYPE, K8S_NAMESPACE, the
# template paths, CLUSTER_NAME, ...). The module injects those vars into the
# worker pod of every package listed here. "containers" is the k8s scope
# itself. Other packages run the same k8s code from their own image with a
# few steps replaced (scheduled-task, containers-datadog); they read the same
# vars, so they belong in this list too.
#
# worker_k8s_packages = ["containers", "scheduled-task"]
#
# Packages not listed here (s3, rds, lambda, ...) do not get these vars.
variable "worker_k8s_packages" {
description = "Package slugs whose worker runs the k8s scope and must receive its env vars (DNS_TYPE, K8S_NAMESPACE, template paths, CLUSTER_NAME, extra_envs). Add every package that runs the k8s scope code, e.g. [\"containers\", \"scheduled-task\"]."
type = list(string)
default = ["containers"]
}

# Name of the Kubernetes cluster the scopes are deployed to.
#
# The k8s scope uses it to find the cluster's OIDC provider when it creates
# the IAM role for a scope. It reaches the workers as the CLUSTER_NAME env var.
# Required when cloud_provider is "aws" (enforced by a precondition in main.tf,
# so a missing value fails at plan time instead of inside create-scope).
#
# cluster_name = module.eks.eks_cluster_name
variable "cluster_name" {
description = "Name of the Kubernetes cluster the scopes run in. Sent to the k8s workers as CLUSTER_NAME; the k8s scope uses it to find the EKS OIDC provider when creating IAM roles. Required when cloud_provider is 'aws'."
type = string
default = ""
}

variable "worker_memory_limit" {
description = "Memory limit for a worker-orchestrated package's pod (packages in var.worker_orchestrated_packages). The chart's own default is small enough to OOM mid-tofu-apply for packages that run real IaC tooling."
type = string
Expand Down
Loading