diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index a842d6391..8b358605c 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -36,6 +36,7 @@ Date: 2026-04-29 | 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | | 9C-1 | Storage-envelope observability: `decrypt_failures_total`, `writes_per_dek`, `value_overhead_bytes`, wired from the storage envelope path through `monitoring.Registry` (§9.2) | shipped | — | | 9C-2 | Sidecar/KEK observability: `active_dek_id{purpose}`, `sidecar_raft_index`, `kek_unwrap_seconds` (§9.2) | shipped | — | +| 9C-3 | Startup KEK-unwrap memoization: collapses the guard + hydration double-unwrap that Stage 9B's KMS providers turned into doubled startup round-trips | shipped | — | | 9C+ | Rotation budget/rewrap/retire/rewrite, `last_proposed_index_per_raft_dek` (needs the §5.4 raft-DEK Wrap path), remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft diff --git a/docs/design/2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md b/docs/design/2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md index 2dd82595b..1e3e97b0c 100644 --- a/docs/design/2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md +++ b/docs/design/2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md @@ -267,7 +267,11 @@ hydrate+bump is gated on an active DEK: ## 5. Resolved decisions and deferred follow-ups -- **Redundant KEK unwrap at startup is deferred to Stage 9.** +- **Redundant KEK unwrap at startup — RESOLVED in Stage 9C.** + Stage 9B shipped the KMS providers, so the condition this deferral + was waiting on arrived. `encryption.StartupUnwrapCache` memoizes + unwraps across the startup phase, collapsing the guard and hydration + passes back to one provider call per wrapped DEK. Original note: `HydrateKeystoreFromSidecar` re-unwraps every wrapped DEK that `CheckStartupGuards` already unwrapped to verify the KEK. For the file-mode KEK (the only provider today) the unwrap is a local AES diff --git a/internal/encryption/kek_startup_cache.go b/internal/encryption/kek_startup_cache.go new file mode 100644 index 000000000..d721d872d --- /dev/null +++ b/internal/encryption/kek_startup_cache.go @@ -0,0 +1,190 @@ +package encryption + +import ( + "crypto/subtle" + "sync" + + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/cockroachdb/errors" +) + +// StartupUnwrapCache memoizes KEK unwraps across the startup phase. +// +// Startup unwraps every wrapped DEK twice: once in the §9.1 guards, to +// prove the configured KEK actually matches the sidecar, and again in +// HydrateKeystoreFromSidecar to populate the keystore. With the +// file-mode KEK that duplication was a local AES operation and cost +// nothing, which is why it was deferred — but Stage 9B shipped the AWS +// KMS, GCP KMS and Vault providers, where every unwrap is a network +// round-trip. The condition that deferral was waiting on has arrived, +// so a node with N wrapped DEKs now makes 2N KMS calls to boot. +// +// The mapping from wrapped bytes to plaintext DEK is deterministic and +// stable, so caching it cannot change a result; it only removes the +// second call. +// +// The cache holds plaintext DEKs, which is why Reset exists: the +// keystore already retains every unretired DEK for the process +// lifetime (historical versions need them), so this adds no new class +// of exposure, but there is no reason to keep a second copy alive past +// hydration. +type StartupUnwrapCache struct { + inner kek.Wrapper + + mu sync.Mutex + entries map[string][]byte + // sealed stops memoizing once startup is done. Without it the + // cache is not merely holding stale entries — it keeps growing, + // retaining a plaintext copy of every DEK a later rotation + // unwraps, for the process lifetime. + sealed bool +} + +// NewStartupUnwrapCache wraps inner, and returns a genuinely nil +// kek.Wrapper when inner is nil so the caller's wiring stays a single +// unconditional call. +// +// The return type is the INTERFACE, not *StartupUnwrapCache. Returning +// a typed nil pointer here would produce a non-nil kek.Wrapper holding +// a nil pointer, and startup decides whether encryption mutators may +// run from `kekWrapper != nil` — a node with no KEK configured would +// report one. +// +// Order matters at the call site: the cache belongs OUTSIDE any +// latency instrumentation, so a cache hit is not recorded as a +// zero-duration KMS round-trip and does not flatten the unwrap +// histogram. +func NewStartupUnwrapCache(inner kek.Wrapper) kek.Wrapper { + if inner == nil { + return nil + } + return &StartupUnwrapCache{inner: inner, entries: make(map[string][]byte)} +} + +// Unwrap returns the memoized plaintext when this exact wrapped blob +// has been unwrapped before, and otherwise delegates. +// +// A failed unwrap is deliberately NOT cached: the failure may be +// transient (a KMS timeout), and caching it would turn one flaky call +// into a permanent startup refusal. +func (c *StartupUnwrapCache) Unwrap(wrapped []byte) ([]byte, error) { + if c == nil { + return nil, ErrKEKNotConfigured + } + if len(wrapped) == 0 { + // Never cache the empty blob: sidecar entries with no wrapped + // material are skipped by the guards, and an empty key would + // collide across purposes. + return c.unwrapUncached(wrapped) + } + + if hit, ok := c.load(wrapped); ok { + return hit, nil + } + dek, err := c.unwrapUncached(wrapped) + if err != nil { + return nil, err + } + c.store(wrapped, dek) + // Hand back a copy so a caller that zeroes or mutates its DEK + // cannot corrupt the cached entry for the next reader. + return append([]byte(nil), dek...), nil +} + +// unwrapUncached delegates to the provider. It wraps the error with +// the provider name so a KMS failure at startup names its source; +// errors.Wrapf preserves Is/As, so the §9.1 guards still match +// ErrKEKMismatch through this decorator. +func (c *StartupUnwrapCache) unwrapUncached(wrapped []byte) ([]byte, error) { + dek, err := c.inner.Unwrap(wrapped) + if err != nil { + return nil, errors.Wrapf(err, "kek %s: unwrap", c.inner.Name()) + } + return dek, nil +} + +func (c *StartupUnwrapCache) load(wrapped []byte) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + dek, ok := c.entries[string(wrapped)] + if !ok { + return nil, false + } + return append([]byte(nil), dek...), true +} + +// store memoizes an unwrap unless the cache has been sealed. After +// sealing, later unwraps (rotation applies) go straight to the +// provider and leave no plaintext behind here. +func (c *StartupUnwrapCache) store(wrapped, dek []byte) { + c.mu.Lock() + defer c.mu.Unlock() + if c.sealed { + return + } + c.entries[string(wrapped)] = append([]byte(nil), dek...) +} + +// Wrap delegates. Wrapping is not memoized: providers may introduce +// fresh randomness per call, so two Wraps of the same DEK legitimately +// differ and a cache would be wrong rather than merely wasteful. +func (c *StartupUnwrapCache) Wrap(dek []byte) ([]byte, error) { + wrapped, err := c.inner.Wrap(dek) + if err != nil { + return nil, errors.Wrapf(err, "kek %s: wrap", c.inner.Name()) + } + return wrapped, nil +} + +// Name reports the underlying provider so logs and the status RPC keep +// showing the real KEK source. +func (c *StartupUnwrapCache) Name() string { return c.inner.Name() } + +// Seal zeroes every cached DEK and stops further memoization. Call it +// once startup has hydrated the keystore. +// +// Clearing alone would not be enough: the same wrapper is retained by +// every applier for the process lifetime, so a cache that kept +// memoizing would accumulate a plaintext copy of every DEK a later +// rotation unwraps. Sealing bounds the window to startup, which is the +// only place the duplicate unwrap it exists to remove occurs. +func (c *StartupUnwrapCache) Seal() { + if c == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + c.sealed = true + for key, dek := range c.entries { + zeroBytes(dek) + delete(c.entries, key) + } +} + +// SealStartupUnwrapCache seals w when it is a StartupUnwrapCache, and +// is a no-op otherwise. Lets the caller seal without knowing whether +// the KEK source was decorated. +func SealStartupUnwrapCache(w kek.Wrapper) { + if cache, ok := w.(*StartupUnwrapCache); ok { + cache.Seal() + } +} + +// Len reports the number of cached entries. Test-facing. +func (c *StartupUnwrapCache) Len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return len(c.entries) +} + +// zeroBytes overwrites key material in place. subtle.ConstantTimeCopy +// is used so the compiler cannot elide the write as dead. +func zeroBytes(b []byte) { + if len(b) == 0 { + return + } + subtle.ConstantTimeCopy(1, b, make([]byte, len(b))) +} diff --git a/internal/encryption/kek_startup_cache_test.go b/internal/encryption/kek_startup_cache_test.go new file mode 100644 index 000000000..436d0dcff --- /dev/null +++ b/internal/encryption/kek_startup_cache_test.go @@ -0,0 +1,212 @@ +package encryption_test + +import ( + "errors" + "sync" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/stretchr/testify/require" +) + +// countingKEK records how many real unwraps reached the provider. +type countingKEK struct { + mu sync.Mutex + unwraps int + wraps int + failNext error +} + +func (k *countingKEK) Name() string { return "counting" } + +func (k *countingKEK) Wrap(dek []byte) ([]byte, error) { + k.mu.Lock() + defer k.mu.Unlock() + k.wraps++ + return append([]byte("w:"), dek...), nil +} + +func (k *countingKEK) Unwrap(wrapped []byte) ([]byte, error) { + k.mu.Lock() + defer k.mu.Unlock() + k.unwraps++ + if k.failNext != nil { + err := k.failNext + k.failNext = nil + return nil, err + } + out := make([]byte, encryption.KeySize) + copy(out, wrapped) + return out, nil +} + +func (k *countingKEK) count() int { + k.mu.Lock() + defer k.mu.Unlock() + return k.unwraps +} + +// TestStartupUnwrapCacheCollapsesTheDuplicateStartupUnwrap is the point +// of the whole type: the §9.1 guards and HydrateKeystoreFromSidecar +// each unwrap every wrapped DEK, which is a doubled KMS round-trip per +// DEK now that Stage 9B shipped the network providers. +func TestStartupUnwrapCacheCollapsesTheDuplicateStartupUnwrap(t *testing.T) { + t.Parallel() + + inner := &countingKEK{} + cache := encryption.NewStartupUnwrapCache(inner) + require.NotNil(t, cache) + + wrapped := [][]byte{[]byte("wrapped-dek-1"), []byte("wrapped-dek-2"), []byte("wrapped-dek-3")} + + // Guard phase. + first := make([][]byte, 0, len(wrapped)) + for _, w := range wrapped { + dek, err := cache.Unwrap(w) + require.NoError(t, err) + first = append(first, dek) + } + require.Equal(t, len(wrapped), inner.count()) + + // Hydration phase over the same sidecar. + for i, w := range wrapped { + dek, err := cache.Unwrap(w) + require.NoError(t, err) + require.Equal(t, first[i], dek, "a cached unwrap must return identical key material") + } + require.Equal(t, len(wrapped), inner.count(), + "the second pass over the same wrapped DEKs must not reach the provider") +} + +// TestStartupUnwrapCacheReturnsANilInterfaceWithoutAKEK guards the +// typed-nil trap. Startup decides whether encryption mutators may run +// from `kekWrapper != nil`, so returning a typed nil pointer would +// make a node with NO KEK configured report that it has one. +func TestStartupUnwrapCacheReturnsANilInterfaceWithoutAKEK(t *testing.T) { + t.Parallel() + + var absent kek.Wrapper + // The assignment to an interface-typed variable is the whole test. + // require.Nil is reflection-based and accepts a typed nil pointer, + // and so does `got == nil` when got is the concrete pointer type — + // both pass even when the bug is present. Only an interface-typed + // comparison distinguishes "no wrapper" from "a wrapper that + // happens to be nil inside", which is what startup branches on. + got := asKEKWrapper(encryption.NewStartupUnwrapCache(absent)) + require.True(t, got == nil, //nolint:testifylint // the interface comparison IS the property under test. + "must be a nil interface, not a typed nil pointer: startup reads kekWrapper != nil to decide whether encryption mutators may run") +} + +// TestStartupUnwrapCacheDoesNotCacheFailures pins that a transient +// provider error is retried. Caching it would turn one flaky KMS call +// into a permanent startup refusal. +func TestStartupUnwrapCacheDoesNotCacheFailures(t *testing.T) { + t.Parallel() + + boom := errors.New("kms timeout") + inner := &countingKEK{failNext: boom} + cache := encryption.NewStartupUnwrapCache(inner) + + _, err := cache.Unwrap([]byte("wrapped")) + require.ErrorIs(t, err, boom) + + dek, err := cache.Unwrap([]byte("wrapped")) + require.NoError(t, err, "a failed unwrap must not be remembered as a failure") + require.Len(t, dek, encryption.KeySize) + require.Equal(t, 2, inner.count()) +} + +// TestStartupUnwrapCacheIsolatesCallersFromEachOther covers a caller +// that zeroes its DEK after use: the cached entry must survive intact +// for the next reader. +func TestStartupUnwrapCacheIsolatesCallersFromEachOther(t *testing.T) { + t.Parallel() + + cache := encryption.NewStartupUnwrapCache(&countingKEK{}) + + first, err := cache.Unwrap([]byte("wrapped")) + require.NoError(t, err) + want := append([]byte(nil), first...) + + // The caller wipes its copy. + for i := range first { + first[i] = 0 + } + + second, err := cache.Unwrap([]byte("wrapped")) + require.NoError(t, err) + require.Equal(t, want, second, "a caller zeroing its DEK must not corrupt the cache") +} + +// TestStartupUnwrapCacheResetDropsKeyMaterial pins that the cache does +// not hold a second copy of every DEK for the process lifetime. +func TestStartupUnwrapCacheResetDropsKeyMaterial(t *testing.T) { + t.Parallel() + + inner := &countingKEK{} + // Routed through asKEKWrapper so this test still compiles if the + // constructor's return type changes, leaving the typed-nil test + // free to fail on its own assertion rather than on a build error. + cache := asKEKWrapper(encryption.NewStartupUnwrapCache(inner)) + concrete, ok := cache.(*encryption.StartupUnwrapCache) + require.True(t, ok) + + _, err := cache.Unwrap([]byte("wrapped")) + require.NoError(t, err) + require.Equal(t, 1, concrete.Len()) + + concrete.Seal() + require.Zero(t, concrete.Len()) + + // After sealing the provider is consulted again... + _, err = cache.Unwrap([]byte("wrapped")) + require.NoError(t, err) + require.Equal(t, 2, inner.count()) + + // ...and, crucially, the result is NOT memoized. The wrapper is + // retained by every applier for the process lifetime, so a cache + // that kept storing would accumulate a plaintext copy of every DEK + // a later rotation unwraps. + require.Zero(t, concrete.Len(), "a sealed cache must not resume memoizing") + _, err = cache.Unwrap([]byte("wrapped")) + require.NoError(t, err) + require.Equal(t, 3, inner.count(), "every post-seal unwrap must reach the provider") +} + +// TestSealStartupUnwrapCacheIsSafeOnAnUndecoratedSource covers the +// production call site, which seals without knowing whether the KEK +// source was decorated at all. +func TestSealStartupUnwrapCacheIsSafeOnAnUndecoratedSource(t *testing.T) { + t.Parallel() + + require.NotPanics(t, func() { + encryption.SealStartupUnwrapCache(nil) + encryption.SealStartupUnwrapCache(&countingKEK{}) + }) +} + +// TestStartupUnwrapCacheDelegatesWrapAndName pins that the decorator +// stays transparent: Wrap must not be memoized, because providers may +// add fresh randomness per call. +func TestStartupUnwrapCacheDelegatesWrapAndName(t *testing.T) { + t.Parallel() + + inner := &countingKEK{} + cache := encryption.NewStartupUnwrapCache(inner) + + require.Equal(t, "counting", cache.Name()) + for range 3 { + _, err := cache.Wrap([]byte("dek")) + require.NoError(t, err) + } + inner.mu.Lock() + defer inner.mu.Unlock() + require.Equal(t, 3, inner.wraps, "Wrap must never be memoized") +} + +// asKEKWrapper forces the interface conversion the production wiring +// performs when it assigns the constructor's result to a kek.Wrapper. +// The conversion is what the typed-nil test observes, so it has to +// happen through a declared interface type rather than by inference. +func asKEKWrapper(w kek.Wrapper) kek.Wrapper { return w } diff --git a/main.go b/main.go index bcd8874e0..3f7555906 100644 --- a/main.go +++ b/main.go @@ -525,6 +525,12 @@ func run() error { ); err != nil { return err } + // Startup has hydrated every keystore, so the KEK unwrap cache has + // served its purpose. Seal it: the same wrapper is retained by + // every applier for the process lifetime, and an unsealed cache + // would keep a plaintext copy of every DEK a later rotation + // unwraps. + encryption.SealStartupUnwrapCache(kekUnwrapper) // Record the active FSM apply sync mode so operators can see on the // /metrics endpoint which durability posture this node is running in. @@ -1855,6 +1861,15 @@ func loadKEKAndRunStartupGuards(unwrapObserver monitoring.KEKUnwrapObserver) (ke // elastickv_encryption_kek_unwrap_seconds empty despite completed // KMS calls. kekWrapper = monitoring.NewTimedKEKUnwrapper(kekWrapper, unwrapObserver) + // Memoize unwraps across the startup phase. The §9.1 guards and + // HydrateKeystoreFromSidecar each unwrap every wrapped DEK, which + // was free under the file KEK but is a doubled network round-trip + // per DEK now that Stage 9B shipped the KMS providers. + // + // The cache sits OUTSIDE the timer on purpose: a cache hit must + // not be recorded as a zero-duration KMS call, which would flatten + // elastickv_encryption_kek_unwrap_seconds. + kekWrapper = encryption.NewStartupUnwrapCache(kekWrapper) if err := encryption.CheckStartupGuards(encryption.StartupConfig{ EncryptionEnabled: *encryptionEnabled, KEKConfigured: kekWrapper != nil,