From 0e7987da4b33157f807c9e1559bd7a965cf06a75 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Fri, 14 Aug 2026 12:38:50 -0700 Subject: [PATCH 1/4] fix(CI): read Vault secrets by field-agnostic import in Windows smoke job The Smoke windows/amd64 job has failed on every run since it was added in #3342 on 2026-05-01, always at the same line: Field "CONFLUENT_CLOUD_EMAIL" not present in secret The job authenticates fine (`vault login` exits 0) and the read itself succeeds, so this is neither a permissions nor a path problem. The linux/arm64 job reads the same secret and passes, which proves the secret exists and holds valid credentials. The difference is how each job reads it. Linux uses `vault-sem-get-secret`, which exports every field the secret contains, so it is agnostic to how those fields are named. Windows cannot use that helper (it is Linux-only), so the read was hand-rolled with `vault kv get -field=` using the *exported* variable names. Those are not the names the fields are stored under -- the helper is what maps them -- so every lookup missed. Mirror the helper instead of hardcoding names: read each secret whole and export every field as an uppercased variable. This removes the coupling to field naming, so it also will not break the next time these secrets are rotated or renamed. Two supporting changes: - Fail immediately if an expected variable is still unset after the import, rather than 20 minutes later inside `go test` as "required environment variable ... is not set". - Log the field names that were imported. Values are never echoed. If the stored names turn out to differ by more than case, this makes the very next run report exactly what they are. Co-Authored-By: Claude Opus 5 --- .semaphore/smoke-tests.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.semaphore/smoke-tests.yml b/.semaphore/smoke-tests.yml index 69404bae1f..84c4961edc 100644 --- a/.semaphore/smoke-tests.yml +++ b/.semaphore/smoke-tests.yml @@ -64,9 +64,36 @@ blocks: # following the pattern from cli-release/.semaphore/4-release-cli.yml). - $Env:VAULT_ADDR = "https://vault.cireops.gcp.internal.confluent.cloud" - vault login -no-print token=$(vault write -field=token "auth/semaphore_self_hosted/login" role="default" jwt="$Env:SEMAPHORE_OIDC_TOKEN") - - $Env:CONFLUENT_CLOUD_EMAIL = (vault kv get -field=CONFLUENT_CLOUD_EMAIL v1/ci/kv/apif/cli/live-testing-data) - - $Env:CONFLUENT_CLOUD_PASSWORD = (vault kv get -field=CONFLUENT_CLOUD_PASSWORD v1/ci/kv/apif/cli/live-testing-data) - - $Env:SLACK_WEBHOOK_URL = (vault kv get -field=SLACK_WEBHOOK_URL v1/ci/kv/apif/cli/slack-notifications-live-testing) + # Read whole secrets and export every field as an uppercased variable, which is + # what `vault-sem-get-secret` does for the linux/arm64 job above. Asking for + # named fields instead (`-field=CONFLUENT_CLOUD_EMAIL`) failed with `Field + # "CONFLUENT_CLOUD_EMAIL" not present in secret`, because the fields are not + # stored under those exact names -- the linux helper is what uppercases them. + # Never echo the values: only field names are ever printed. + - | + function Import-VaultSecret($Path) { + $secret = vault kv get -format=json $Path | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { throw "failed to read Vault secret $Path" } + + # KV v2 nests fields under .data.data; KV v1 puts them directly on .data. + $fields = if ($secret.data.PSObject.Properties.Name -contains "data") { $secret.data.data } else { $secret.data } + + foreach ($field in $fields.PSObject.Properties) { + [Environment]::SetEnvironmentVariable($field.Name.ToUpper(), $field.Value) + } + Write-Host "Imported $Path fields: $(($fields.PSObject.Properties.Name | Sort-Object) -join ', ')" + } + + Import-VaultSecret v1/ci/kv/apif/cli/live-testing-data + Import-VaultSecret v1/ci/kv/apif/cli/slack-notifications-live-testing + + # Fail here rather than 20 minutes later inside `go test`, which would only + # report "required environment variable ... is not set". + foreach ($name in @("CONFLUENT_CLOUD_EMAIL", "CONFLUENT_CLOUD_PASSWORD", "SLACK_WEBHOOK_URL")) { + if (-not [Environment]::GetEnvironmentVariable($name)) { + throw "$name is not set after importing Vault secrets" + } + } # Install Go (matches the pattern in semaphore.yml; chocolatey is community-maintained) - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -OutFile Go.zip -Uri https://go.dev/dl/go$(Get-Content .go-version).windows-amd64.zip -UseBasicParsing From ab034f4256509731c4925140cae172409f4f28fb Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Fri, 14 Aug 2026 13:23:11 -0700 Subject: [PATCH 2/4] fix(CI): narrow Vault export and check exit code before parsing Two hardening changes from a review of the previous commit. Check vault's exit code before piping to ConvertFrom-Json. On a partial or malformed response the parse would run first, and PowerShell's JSON errors can echo the offending input, which may contain secret values. Export only the three variables the job needs, matched case-insensitively, instead of exporting every field in both secrets. The previous approach mirrored `vault-sem-get-secret`, but it also placed every unrelated field in those secrets into the environment inherited by `go test`, the CLI under test, and the metric emitter -- a wider disclosure surface than the job requires. Matching case-insensitively keeps the fix independent of how the fields are cased, which was the point of reading them by hand. Note this does not change what is read from Vault: `vault kv get -field=` has no server-side projection, so the whole secret was already fetched either way. Only what reaches the environment changes. The explicit post-import guard is now redundant and has been dropped, since a missing or empty field throws at the point of export. On a mismatch the error lists the available field names, so a single run still reveals them. Values are never echoed, and the success path prints nothing at all. Co-Authored-By: Claude Opus 5 --- .semaphore/smoke-tests.yml | 49 ++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/.semaphore/smoke-tests.yml b/.semaphore/smoke-tests.yml index 84c4961edc..a10419f282 100644 --- a/.semaphore/smoke-tests.yml +++ b/.semaphore/smoke-tests.yml @@ -64,36 +64,45 @@ blocks: # following the pattern from cli-release/.semaphore/4-release-cli.yml). - $Env:VAULT_ADDR = "https://vault.cireops.gcp.internal.confluent.cloud" - vault login -no-print token=$(vault write -field=token "auth/semaphore_self_hosted/login" role="default" jwt="$Env:SEMAPHORE_OIDC_TOKEN") - # Read whole secrets and export every field as an uppercased variable, which is - # what `vault-sem-get-secret` does for the linux/arm64 job above. Asking for - # named fields instead (`-field=CONFLUENT_CLOUD_EMAIL`) failed with `Field - # "CONFLUENT_CLOUD_EMAIL" not present in secret`, because the fields are not - # stored under those exact names -- the linux helper is what uppercases them. - # Never echo the values: only field names are ever printed. + # Match fields case-insensitively rather than by exact name. Asking for + # `-field=CONFLUENT_CLOUD_EMAIL` failed with `Field "CONFLUENT_CLOUD_EMAIL" not + # present in secret`, because the fields are not stored under the names they are + # exported as -- on linux/arm64 above, `vault-sem-get-secret` is what maps them. + # Only the three variables the job needs are exported, so unrelated fields in + # these secrets never reach the environment of `go test` and its children. + # Values are never echoed; only field names appear, and only on failure. - | - function Import-VaultSecret($Path) { - $secret = vault kv get -format=json $Path | ConvertFrom-Json + function Get-VaultSecret($Path) { + # Check the exit code before parsing: on a partial or malformed response, + # ConvertFrom-Json can echo the offending input, which may hold secrets. + $json = vault kv get -format=json $Path if ($LASTEXITCODE -ne 0) { throw "failed to read Vault secret $Path" } + $secret = $json | ConvertFrom-Json + # KV v2 nests fields under .data.data; KV v1 puts them directly on .data. - $fields = if ($secret.data.PSObject.Properties.Name -contains "data") { $secret.data.data } else { $secret.data } + if ($secret.data.PSObject.Properties.Name -contains "data") { return $secret.data.data } + return $secret.data + } - foreach ($field in $fields.PSObject.Properties) { - [Environment]::SetEnvironmentVariable($field.Name.ToUpper(), $field.Value) + function Export-VaultField($Fields, $Name) { + $field = $Fields.PSObject.Properties | Where-Object { $_.Name -ieq $Name } | Select-Object -First 1 + if (-not $field) { + throw "no field matching '$Name' in secret (available: $(($Fields.PSObject.Properties.Name | Sort-Object) -join ', '))" } - Write-Host "Imported $Path fields: $(($fields.PSObject.Properties.Name | Sort-Object) -join ', ')" - } + if ([string]::IsNullOrEmpty($field.Value)) { throw "field matching '$Name' is empty" } - Import-VaultSecret v1/ci/kv/apif/cli/live-testing-data - Import-VaultSecret v1/ci/kv/apif/cli/slack-notifications-live-testing + [Environment]::SetEnvironmentVariable($Name, $field.Value) + } # Fail here rather than 20 minutes later inside `go test`, which would only # report "required environment variable ... is not set". - foreach ($name in @("CONFLUENT_CLOUD_EMAIL", "CONFLUENT_CLOUD_PASSWORD", "SLACK_WEBHOOK_URL")) { - if (-not [Environment]::GetEnvironmentVariable($name)) { - throw "$name is not set after importing Vault secrets" - } - } + $liveTestingData = Get-VaultSecret v1/ci/kv/apif/cli/live-testing-data + Export-VaultField $liveTestingData "CONFLUENT_CLOUD_EMAIL" + Export-VaultField $liveTestingData "CONFLUENT_CLOUD_PASSWORD" + + $slackNotifications = Get-VaultSecret v1/ci/kv/apif/cli/slack-notifications-live-testing + Export-VaultField $slackNotifications "SLACK_WEBHOOK_URL" # Install Go (matches the pattern in semaphore.yml; chocolatey is community-maintained) - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -OutFile Go.zip -Uri https://go.dev/dl/go$(Get-Content .go-version).windows-amd64.zip -UseBasicParsing From 39d30dfaa245bcc71dd167f78e2b219d603dd5b5 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Fri, 14 Aug 2026 13:37:53 -0700 Subject: [PATCH 3/4] fix(CI): read the "script" field, correcting the root cause The previous two commits diagnosed this wrong. They assumed the secret stored each credential in its own field under a different casing, and matched fields case-insensitively. That would not have fixed the job. Checking how the linux/arm64 job's helper actually behaves settles it. It requests one specific field, named "script", and sources its contents as a shell script. It performs no mapping or renaming of variable names, and the vault helper it calls beforehand only handles OIDC auth. So these secrets keep everything in that single "script" field, whose value is a shell script of `export NAME=value` lines. No CONFLUENT_CLOUD_EMAIL field exists in any casing, which is precisely what Vault reported. Read the "script" field and parse the assignments out of it, since PowerShell cannot source a shell script. The parser tolerates `export` or bare assignments, surrounding single or double quotes, values containing `=`, and skips comments and blank lines. The properties from the previous commit are kept: the exit code is checked before the output is used, only the variables the job needs are exported, no values are ever echoed, and a missing or empty variable throws immediately while listing the names the script does define. Co-Authored-By: Claude Opus 5 --- .semaphore/smoke-tests.yml | 67 ++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/.semaphore/smoke-tests.yml b/.semaphore/smoke-tests.yml index a10419f282..bb66b82b37 100644 --- a/.semaphore/smoke-tests.yml +++ b/.semaphore/smoke-tests.yml @@ -64,45 +64,50 @@ blocks: # following the pattern from cli-release/.semaphore/4-release-cli.yml). - $Env:VAULT_ADDR = "https://vault.cireops.gcp.internal.confluent.cloud" - vault login -no-print token=$(vault write -field=token "auth/semaphore_self_hosted/login" role="default" jwt="$Env:SEMAPHORE_OIDC_TOKEN") - # Match fields case-insensitively rather than by exact name. Asking for + # These secrets keep everything in a single field named "script", whose value is + # a shell script of `export NAME=value` lines. That is why asking for # `-field=CONFLUENT_CLOUD_EMAIL` failed with `Field "CONFLUENT_CLOUD_EMAIL" not - # present in secret`, because the fields are not stored under the names they are - # exported as -- on linux/arm64 above, `vault-sem-get-secret` is what maps them. - # Only the three variables the job needs are exported, so unrelated fields in - # these secrets never reach the environment of `go test` and its children. - # Values are never echoed; only field names appear, and only on failure. + # present in secret` -- no such field exists. The linux/arm64 job above sources + # that script; PowerShell cannot, so parse the assignments out of it instead. + # Only the variables the job needs are exported, so anything else these secrets + # define never reaches the environment of `go test` and its children. + # Values are never echoed; only variable names appear, and only on failure. - | - function Get-VaultSecret($Path) { - # Check the exit code before parsing: on a partial or malformed response, - # ConvertFrom-Json can echo the offending input, which may hold secrets. - $json = vault kv get -format=json $Path + function Import-VaultScript($Path, $Names) { + # Check the exit code before touching the output: on a partial response, + # downstream errors could echo the input, which holds secrets. + $script = vault kv get -field=script $Path if ($LASTEXITCODE -ne 0) { throw "failed to read Vault secret $Path" } - $secret = $json | ConvertFrom-Json - - # KV v2 nests fields under .data.data; KV v1 puts them directly on .data. - if ($secret.data.PSObject.Properties.Name -contains "data") { return $secret.data.data } - return $secret.data - } - - function Export-VaultField($Fields, $Name) { - $field = $Fields.PSObject.Properties | Where-Object { $_.Name -ieq $Name } | Select-Object -First 1 - if (-not $field) { - throw "no field matching '$Name' in secret (available: $(($Fields.PSObject.Properties.Name | Sort-Object) -join ', '))" + $defined = @{} + foreach ($line in ($script -split "\r?\n")) { + if ($line -notmatch '^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$') { continue } + + $name = $Matches[1] + $value = $Matches[2].Trim() + + # Strip one layer of surrounding quotes, as `source` would. + foreach ($quote in '"', "'") { + if ($value.Length -ge 2 -and $value.StartsWith($quote) -and $value.EndsWith($quote)) { + $value = $value.Substring(1, $value.Length - 2) + break + } + } + $defined[$name] = $value } - if ([string]::IsNullOrEmpty($field.Value)) { throw "field matching '$Name' is empty" } - [Environment]::SetEnvironmentVariable($Name, $field.Value) + # Fail here rather than 20 minutes later inside `go test`, which would only + # report "required environment variable ... is not set". + foreach ($name in $Names) { + if (-not $defined.ContainsKey($name) -or [string]::IsNullOrEmpty($defined[$name])) { + throw "secret $Path does not define $name (defines: $(($defined.Keys | Sort-Object) -join ', '))" + } + [Environment]::SetEnvironmentVariable($name, $defined[$name]) + } } - # Fail here rather than 20 minutes later inside `go test`, which would only - # report "required environment variable ... is not set". - $liveTestingData = Get-VaultSecret v1/ci/kv/apif/cli/live-testing-data - Export-VaultField $liveTestingData "CONFLUENT_CLOUD_EMAIL" - Export-VaultField $liveTestingData "CONFLUENT_CLOUD_PASSWORD" - - $slackNotifications = Get-VaultSecret v1/ci/kv/apif/cli/slack-notifications-live-testing - Export-VaultField $slackNotifications "SLACK_WEBHOOK_URL" + Import-VaultScript v1/ci/kv/apif/cli/live-testing-data @("CONFLUENT_CLOUD_EMAIL", "CONFLUENT_CLOUD_PASSWORD") + Import-VaultScript v1/ci/kv/apif/cli/slack-notifications-live-testing @("SLACK_WEBHOOK_URL") # Install Go (matches the pattern in semaphore.yml; chocolatey is community-maintained) - $ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest -OutFile Go.zip -Uri https://go.dev/dl/go$(Get-Content .go-version).windows-amd64.zip -UseBasicParsing From 011739339be31ead88ce48e8df9441ed2dbd48b9 Mon Sep 17 00:00:00 2001 From: Cynthia Qin Date: Fri, 14 Aug 2026 13:51:52 -0700 Subject: [PATCH 4/4] fix(CI): match assignments positively and rename a confusing variable Two cleanups from re-reviewing the whole change; no behavior change intended. The parse loop tested `-notmatch` and then read `$Matches` after the negated test. PowerShell does populate `$Matches` whenever the underlying regex matches, regardless of which operator was used, so this worked -- but it depends on non-obvious behavior that cannot be verified from a machine without PowerShell. Use a positive `-match` and nest the body instead, which needs no such assumption and reads more directly. Rename `$script` to `$exports`. `$script` is legal but sits one colon away from PowerShell's `$script:` scope modifier, which is needlessly confusing here. Also note in a comment that the hashtable lookups are case-insensitive, since PowerShell hashtables are by default. That means the import still succeeds if the script declares these names under a different case -- worth stating, as an earlier revision of this branch wrongly believed case was the whole problem. Co-Authored-By: Claude Opus 5 --- .semaphore/smoke-tests.yml | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/.semaphore/smoke-tests.yml b/.semaphore/smoke-tests.yml index bb66b82b37..3381210b0d 100644 --- a/.semaphore/smoke-tests.yml +++ b/.semaphore/smoke-tests.yml @@ -76,24 +76,26 @@ blocks: function Import-VaultScript($Path, $Names) { # Check the exit code before touching the output: on a partial response, # downstream errors could echo the input, which holds secrets. - $script = vault kv get -field=script $Path + $exports = vault kv get -field=script $Path if ($LASTEXITCODE -ne 0) { throw "failed to read Vault secret $Path" } + # Hashtable keys are case-insensitive, so the lookups below still succeed if + # the script happens to declare these names under a different case. $defined = @{} - foreach ($line in ($script -split "\r?\n")) { - if ($line -notmatch '^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$') { continue } - - $name = $Matches[1] - $value = $Matches[2].Trim() - - # Strip one layer of surrounding quotes, as `source` would. - foreach ($quote in '"', "'") { - if ($value.Length -ge 2 -and $value.StartsWith($quote) -and $value.EndsWith($quote)) { - $value = $value.Substring(1, $value.Length - 2) - break + foreach ($line in ($exports -split "\r?\n")) { + if ($line -match '^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$') { + $name = $Matches[1] + $value = $Matches[2].Trim() + + # Strip one layer of surrounding quotes, as `source` would. + foreach ($quote in '"', "'") { + if ($value.Length -ge 2 -and $value.StartsWith($quote) -and $value.EndsWith($quote)) { + $value = $value.Substring(1, $value.Length - 2) + break + } } + $defined[$name] = $value } - $defined[$name] = $value } # Fail here rather than 20 minutes later inside `go test`, which would only