From 9532d801f36e55c29fad955817a6b3460e63905b Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Thu, 10 Sep 2026 14:00:06 -0300 Subject: [PATCH 1/4] fix(k8s): ingress render with both custom domains and additional ports --- CHANGELOG.md | 3 + .../templates/blue-green-ingress.yaml.tpl | 12 +- .../templates/initial-ingress.yaml.tpl | 6 +- .../ingress_domains_additional_ports.bats | 123 ++++++++++++++++++ 4 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 k8s/deployment/tests/ingress_domains_additional_ports.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 948ece76..0e1dee2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +- Fix: k8s scopes that have both a custom domain and additional ports now deploy, instead of failing with "Failed to build ingress template" + ## [1.16.3] - 2026-09-08 - Add: remove resource level restriction for `elasticloadbalancing:Describe*` permissions as AWS does not support it. - Fix: k8s deployment logs no longer show a "np_trace_flush: command not found" error when tracing is disabled diff --git a/k8s/deployment/templates/blue-green-ingress.yaml.tpl b/k8s/deployment/templates/blue-green-ingress.yaml.tpl index c35f53fe..b937725f 100644 --- a/k8s/deployment/templates/blue-green-ingress.yaml.tpl +++ b/k8s/deployment/templates/blue-green-ingress.yaml.tpl @@ -83,6 +83,8 @@ spec: {{ if .scope.capabilities.additional_ports }} {{ range .scope.capabilities.additional_ports }} +{{- $port := .port }} +{{- $port_type := .type }} --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -192,20 +194,20 @@ spec: - host: {{ .name }} http: paths: - {{ if eq $.type "HTTP" }} - - path: /{{ $.port }} + {{ if eq $port_type "HTTP" }} + - path: /{{ $port }} pathType: Prefix backend: service: - name: bg-deployment-{{ if eq $.type "HTTP" }}http{{ else }}grpc{{ end }}-{{ $.port }} + name: bg-deployment-{{ if eq $port_type "HTTP" }}http{{ else }}grpc{{ end }}-{{ $port }} port: name: use-annotation - {{ else if eq $.type "GRPC" }} + {{ else if eq $port_type "GRPC" }} - path: / pathType: Prefix backend: service: - name: bg-deployment-{{ if eq $.type "HTTP" }}http{{ else }}grpc{{ end }}-{{ $.port }} + name: bg-deployment-{{ if eq $port_type "HTTP" }}http{{ else }}grpc{{ end }}-{{ $port }} port: name: use-annotation {{ end }} diff --git a/k8s/deployment/templates/initial-ingress.yaml.tpl b/k8s/deployment/templates/initial-ingress.yaml.tpl index 088a1eaf..1524b615 100644 --- a/k8s/deployment/templates/initial-ingress.yaml.tpl +++ b/k8s/deployment/templates/initial-ingress.yaml.tpl @@ -77,6 +77,8 @@ spec: {{- end }} {{ if .scope.capabilities.additional_ports }} {{ range .scope.capabilities.additional_ports }} +{{- $port := .port }} +{{- $port_type := .type }} --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -155,9 +157,9 @@ spec: pathType: Prefix backend: service: - name: d-{{ $.scope.id }}-{{ $.deployment.id }}-{{ if eq .type "HTTP" }}http{{ else }}grpc{{ end }}-{{ .port }} + name: d-{{ $.scope.id }}-{{ $.deployment.id }}-{{ if eq $port_type "HTTP" }}http{{ else }}grpc{{ end }}-{{ $port }} port: - number: {{ .port }} + number: {{ $port }} {{- end }} {{ end }} {{ end }} diff --git a/k8s/deployment/tests/ingress_domains_additional_ports.bats b/k8s/deployment/tests/ingress_domains_additional_ports.bats new file mode 100644 index 00000000..1cdf28b7 --- /dev/null +++ b/k8s/deployment/tests/ingress_domains_additional_ports.bats @@ -0,0 +1,123 @@ +#!/usr/bin/env bats + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + export TPL_DIR="$PROJECT_ROOT/k8s/deployment/templates" + export INITIAL="$TPL_DIR/initial-ingress.yaml.tpl" + export BLUE_GREEN="$TPL_DIR/blue-green-ingress.yaml.tpl" +} + +_context() { + cat <<'JSON' +{ + "account": {"id": "acc1", "slug": "acct"}, + "namespace": {"id": "ns1", "slug": "nsps"}, + "application": {"id": "app1", "slug": "appslug"}, + "scope": { + "id": "scope-123", + "slug": "scopeslug", + "domain": "platform.example.com", + "domains": [ + {"id": "dom-1", "name": "custom-one.example.com", "status": "active", "type": "scope"}, + {"id": "dom-2", "name": "custom-two.example.com", "status": "active", "type": "scope"} + ], + "capabilities": { + "main_http_port": 8080, + "additional_ports": [ + {"port": 8081, "type": "HTTP", "traffic_manager_port": 18081}, + {"port": 9012, "type": "GRPC", "traffic_manager_port": 19012} + ] + } + }, + "deployment": {"id": "deploy-456", "strategy_data": {"desired_switched_traffic": 50}}, + "blue_deployment_id": "deploy-123", + "blue_additional_port_services": {"http-8081": true, "grpc-9012": true}, + "k8s_namespace": "ns-test", + "k8s_modifiers": {}, + "alb_name": "k8s-test-alb", + "ingress_visibility": "internal", + "main_http_port": 8080 +} +JSON +} + +_render() { + local tpl="$1" + local ctx="$BATS_TEST_TMPDIR/ctx.json" + _context > "$ctx" + gomplate -c .="$ctx" -f "$tpl" +} + +_backend() { + local rendered="$1" ingress="$2" host="$3" + echo "$rendered" | yq -N "select(.metadata.name == \"$ingress\") | .spec.rules[] | select(.host == \"$host\") | .http.paths[0].backend.service | .name + \":\" + (.port.number // .port.name | tostring)" +} + +_path() { + local rendered="$1" ingress="$2" host="$3" + echo "$rendered" | yq -N "select(.metadata.name == \"$ingress\") | .spec.rules[] | select(.host == \"$host\") | .http.paths[0].path" +} + +@test "initial-ingress: renders when a scope has both custom domains and additional ports" { + run _render "$INITIAL" + [ "$status" -eq 0 ] +} + +@test "initial-ingress: custom domains route each additional port to its own service" { + rendered=$(_render "$INITIAL") + + for host in custom-one.example.com custom-two.example.com; do + assert_equal "d-scope-123-deploy-456-http-8081:8081" \ + "$(_backend "$rendered" k-8-s-scopeslug-scope-123-http-8081-internal "$host")" + assert_equal "d-scope-123-deploy-456-grpc-9012:9012" \ + "$(_backend "$rendered" k-8-s-scopeslug-scope-123-grpc-9012-internal "$host")" + done +} + +@test "initial-ingress: custom domain rules match the platform domain rule" { + rendered=$(_render "$INITIAL") + + for ingress in k-8-s-scopeslug-scope-123-http-8081-internal k-8-s-scopeslug-scope-123-grpc-9012-internal; do + platform=$(_backend "$rendered" "$ingress" platform.example.com) + for host in custom-one.example.com custom-two.example.com; do + assert_equal "$platform" "$(_backend "$rendered" "$ingress" "$host")" + done + done +} + +@test "initial-ingress: every rendered document is valid yaml with a name" { + rendered=$(_render "$INITIAL") + + names=$(echo "$rendered" | yq -N '.metadata.name') + assert_equal 3 "$(echo "$names" | grep -c .)" + if echo "$names" | grep -q 'null'; then + echo "a rendered document lost its metadata.name: $rendered" + return 1 + fi +} + +@test "blue-green-ingress: renders when a scope has both custom domains and additional ports" { + run _render "$BLUE_GREEN" + [ "$status" -eq 0 ] +} + +@test "blue-green-ingress: custom domains route each additional port through its own annotation action" { + rendered=$(_render "$BLUE_GREEN") + + for host in custom-one.example.com custom-two.example.com; do + assert_equal "bg-deployment-http-8081:use-annotation" \ + "$(_backend "$rendered" k-8-s-scopeslug-scope-123-http-8081-internal "$host")" + assert_equal "bg-deployment-grpc-9012:use-annotation" \ + "$(_backend "$rendered" k-8-s-scopeslug-scope-123-grpc-9012-internal "$host")" + done +} + +@test "blue-green-ingress: each additional port keeps its own path shape on custom domains" { + rendered=$(_render "$BLUE_GREEN") + + for host in custom-one.example.com custom-two.example.com; do + assert_equal "/8081" "$(_path "$rendered" k-8-s-scopeslug-scope-123-http-8081-internal "$host")" + assert_equal "/" "$(_path "$rendered" k-8-s-scopeslug-scope-123-grpc-9012-internal "$host")" + done +} From 0e0afd06ad28aefc1d8d7badf8b53dc932c1b089 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Fri, 11 Sep 2026 14:32:56 -0300 Subject: [PATCH 2/4] fix(k8s): honor health_check.enabled in the deployment template --- CHANGELOG.md | 1 + azure-aro/specs/service-spec.json.tpl | 3 +- azure/specs/service-spec.json.tpl | 3 +- k8s/deployment/templates/deployment.yaml.tpl | 9 ++ .../tests/health_check_disabled.bats | 130 ++++++++++++++++++ k8s/specs/service-spec.json.tpl | 3 +- 6 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 k8s/deployment/tests/health_check_disabled.bats diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e1dee2c..159a3ba1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +- Fix: turning off the health check on a k8s, azure or azure-aro scope now removes the liveness, readiness and startup probes from every container of the pod - Fix: k8s scopes that have both a custom domain and additional ports now deploy, instead of failing with "Failed to build ingress template" ## [1.16.3] - 2026-09-08 diff --git a/azure-aro/specs/service-spec.json.tpl b/azure-aro/specs/service-spec.json.tpl index 8de657f9..5b13b495 100644 --- a/azure-aro/specs/service-spec.json.tpl +++ b/azure-aro/specs/service-spec.json.tpl @@ -440,7 +440,8 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true + "default":true, + "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" }, "period_seconds":{ "type":"integer", diff --git a/azure/specs/service-spec.json.tpl b/azure/specs/service-spec.json.tpl index 1a4e705d..d70c00b1 100644 --- a/azure/specs/service-spec.json.tpl +++ b/azure/specs/service-spec.json.tpl @@ -442,7 +442,8 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true + "default":true, + "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" }, "period_seconds":{ "type":"integer", diff --git a/k8s/deployment/templates/deployment.yaml.tpl b/k8s/deployment/templates/deployment.yaml.tpl index 0e513c1e..528cedd6 100644 --- a/k8s/deployment/templates/deployment.yaml.tpl +++ b/k8s/deployment/templates/deployment.yaml.tpl @@ -21,6 +21,7 @@ initialDelaySeconds: {{ .healthCheck.initial_delay_seconds }} successThreshold: 1 {{- end }} +{{- $healthCheckEnabled := or (not (has .scope.capabilities.health_check "enabled")) .scope.capabilities.health_check.enabled }} apiVersion: apps/v1 kind: Deployment @@ -147,6 +148,7 @@ spec: memory: {{ .container_memory_in_memory }}Mi requests: cpu: 31m + {{- if $healthCheckEnabled }} livenessProbe: {{- if and (has .scope.capabilities.health_check "type") (eq .scope.capabilities.health_check.type "TCP") }} {{- template "probe.tcp" dict "healthCheck" .scope.capabilities.health_check "traffic_port" .main_traffic_manager_port "app_port" .main_http_port }} @@ -171,6 +173,7 @@ spec: {{- end }} {{- template "probe.base" dict "healthCheck" .scope.capabilities.health_check }} failureThreshold: 90 + {{- end }} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: Always @@ -202,6 +205,7 @@ spec: memory: {{ $.container_memory_in_memory }}Mi requests: cpu: 31m + {{- if $healthCheckEnabled }} livenessProbe: grpc: port: {{ .traffic_manager_port }} @@ -226,6 +230,7 @@ spec: initialDelaySeconds: {{ $.scope.capabilities.health_check.initial_delay_seconds }} successThreshold: 1 failureThreshold: 90 + {{- end }} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: Always @@ -256,6 +261,7 @@ spec: memory: {{ $.container_memory_in_memory }}Mi requests: cpu: 31m + {{- if $healthCheckEnabled }} livenessProbe: httpGet: path: {{ $.scope.capabilities.health_check.path }} @@ -283,6 +289,7 @@ spec: initialDelaySeconds: {{ $.scope.capabilities.health_check.initial_delay_seconds }} successThreshold: 1 failureThreshold: 90 + {{- end }} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File imagePullPolicy: Always @@ -323,6 +330,7 @@ spec: requests: cpu: {{ .scope.capabilities.cpu_millicores }}m memory: {{ .scope.capabilities.ram_memory }}Mi + {{- if $healthCheckEnabled }} livenessProbe: {{- if and (has .scope.capabilities.health_check "type") (eq .scope.capabilities.health_check.type "TCP") }} {{- template "probe.app_tcp" dict "port" .main_http_port }} @@ -347,6 +355,7 @@ spec: {{- end }} {{- template "probe.base" dict "healthCheck" .scope.capabilities.health_check }} failureThreshold: 90 + {{- end }} lifecycle: preStop: exec: diff --git a/k8s/deployment/tests/health_check_disabled.bats b/k8s/deployment/tests/health_check_disabled.bats new file mode 100644 index 00000000..f420ad1d --- /dev/null +++ b/k8s/deployment/tests/health_check_disabled.bats @@ -0,0 +1,130 @@ +#!/usr/bin/env bats + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + export DEPLOYMENT="$PROJECT_ROOT/k8s/deployment/templates/deployment.yaml.tpl" +} + +_context() { + local health_check="$1" + cat < "$ctx" + gomplate -c .="$ctx" -f "$DEPLOYMENT" +} + +_probe_count() { + local rendered="$1" container="$2" + echo "$rendered" | yq -N ".spec.template.spec.containers[] | select(.name == \"$container\") | [.livenessProbe, .readinessProbe, .startupProbe] | map(select(. != null)) | length" +} + +_assert_probes_on_every_container() { + local rendered="$1" expected="$2" + local container actual + for container in http application http-8081 grpc-9012; do + actual=$(_probe_count "$rendered" "$container") + if [ "$actual" != "$expected" ]; then + echo "container '$container': expected $expected probes, got $actual" + return 1 + fi + done +} + +@test "health check disabled: no container gets liveness, readiness or startup probes" { + rendered=$(_render "$(_health_check false HTTP)") + + _assert_probes_on_every_container "$rendered" 0 +} + +@test "health check disabled with TCP type: no container gets probes either" { + rendered=$(_render "$(_health_check false TCP)") + + _assert_probes_on_every_container "$rendered" 0 +} + +@test "health check enabled: every container keeps its three probes" { + rendered=$(_render "$(_health_check true HTTP)") + + _assert_probes_on_every_container "$rendered" 3 +} + +@test "scope predating the enabled flag: every container keeps its three probes" { + rendered=$(_render '{"type": "HTTP", "path": "/health", "timeout_seconds": 1, "period_seconds": 5, "initial_delay_seconds": 5}') + + _assert_probes_on_every_container "$rendered" 3 +} + +@test "health check disabled: containers and their ports survive" { + rendered=$(_render "$(_health_check false HTTP)") + + assert_equal "$(echo "$rendered" | yq -N '[.spec.template.spec.containers[].name] | join(",")')" \ + "http,http-8081,grpc-9012,application" + assert_equal "$(echo "$rendered" | yq -N '[.spec.template.spec.containers[] | select(.name == "application") | .ports[].containerPort] | join(",")')" \ + "8080,8081,9012" + assert_equal "$(echo "$rendered" | yq -N '.spec.template.spec.containers[] | select(.name == "http") | .ports[0].containerPort')" \ + "80" +} + +@test "health check disabled: the application keeps its graceful shutdown hook" { + rendered=$(_render "$(_health_check false HTTP)") + + assert_equal "$(echo "$rendered" | yq -N '.spec.template.spec.containers[] | select(.name == "application") | .lifecycle.preStop.exec.command | join(" ")')" \ + "/bin/sleep 16" +} diff --git a/k8s/specs/service-spec.json.tpl b/k8s/specs/service-spec.json.tpl index 60763b89..468a192a 100644 --- a/k8s/specs/service-spec.json.tpl +++ b/k8s/specs/service-spec.json.tpl @@ -481,7 +481,8 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true + "default":true, + "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" }, "period_seconds":{ "type":"integer", From cdf200800b10de13c6f7e50856b31640412c94a4 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Fri, 11 Sep 2026 15:06:15 -0300 Subject: [PATCH 3/4] chore(k8s): drop the health_check.enabled description --- azure-aro/specs/service-spec.json.tpl | 3 +-- azure/specs/service-spec.json.tpl | 3 +-- k8s/specs/service-spec.json.tpl | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/azure-aro/specs/service-spec.json.tpl b/azure-aro/specs/service-spec.json.tpl index 5b13b495..8de657f9 100644 --- a/azure-aro/specs/service-spec.json.tpl +++ b/azure-aro/specs/service-spec.json.tpl @@ -440,8 +440,7 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true, - "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" + "default":true }, "period_seconds":{ "type":"integer", diff --git a/azure/specs/service-spec.json.tpl b/azure/specs/service-spec.json.tpl index d70c00b1..1a4e705d 100644 --- a/azure/specs/service-spec.json.tpl +++ b/azure/specs/service-spec.json.tpl @@ -442,8 +442,7 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true, - "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" + "default":true }, "period_seconds":{ "type":"integer", diff --git a/k8s/specs/service-spec.json.tpl b/k8s/specs/service-spec.json.tpl index 468a192a..60763b89 100644 --- a/k8s/specs/service-spec.json.tpl +++ b/k8s/specs/service-spec.json.tpl @@ -481,8 +481,7 @@ "enabled":{ "type":"boolean", "title":"Enable Health Check", - "default":true, - "description":"When off, instances are considered available as soon as the process starts and nullplatform stops restarting the ones that fail to respond" + "default":true }, "period_seconds":{ "type":"integer", From ab8fb2379ef6ecc27c718b57da928810d2805df8 Mon Sep 17 00:00:00 2001 From: Federico Maleh Date: Fri, 11 Sep 2026 16:10:54 -0300 Subject: [PATCH 4/4] Release 1.17.0 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 159a3ba1..6bba1268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.17.0] - 2026-09-11 - Fix: turning off the health check on a k8s, azure or azure-aro scope now removes the liveness, readiness and startup probes from every container of the pod - Fix: k8s scopes that have both a custom domain and additional ports now deploy, instead of failing with "Failed to build ingress template"