From cbb6e954f94ab10a5b750a7eef6a62f6a08b8b7c Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 11 Sep 2026 00:35:04 +0900 Subject: [PATCH 1/2] =?UTF-8?q?encryption:=20add=20the=20=C2=A75.4=20rewri?= =?UTF-8?q?te-job=20decision=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 9C-6. Two pure decisions the rotation rewrite job needs, split out from its execution half so they are testable without a store, a clock, or a Raft group. ClassifyMVCCRewrite decides whether one MVCC version must be re-encrypted. Two properties it exists to enforce: - The iteration unit is (user_key, version_ts), not user_key. A rewrite that only touches the newest version leaves history still readable under the retiring DEK, so retirement can never become eligible (§5.4's storage criteria require the rewrite cursor to pass every value). - migrateCleartext is a separate input from retiringKeyID, so a routine rotation never silently begins encrypting data the operator did not opt into. Encrypting existing cleartext is the §7.1 migration, a different operation with a different rollback story. RewriteThrottle implements the --rate=N MiB/s inter-batch yield. The delay is computed from cumulative bytes against cumulative elapsed time rather than per batch: a per-batch rule lets a job that stalled for unrelated reasons bank the idle time and then burst far above the configured rate, which is the write-amplification spike against live traffic that the budget exists to prevent. Claude-Session: https://claude.ai/code/session_013rNHooj7NF3giihWVba8QE --- ...6_04_29_partial_data_at_rest_encryption.md | 3 +- internal/encryption/rewrite_plan.go | 167 ++++++++++++++ internal/encryption/rewrite_plan_test.go | 216 ++++++++++++++++++ 3 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 internal/encryption/rewrite_plan.go create mode 100644 internal/encryption/rewrite_plan_test.go diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0588b2805..01dc17211 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -34,7 +34,8 @@ Date: 2026-04-29 | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) | | 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` | | 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | -| 9C+ | Rotation budget/rewrap/retire/rewrite, metrics, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | +| 9C-6 | §5.4 rewrite-job decisions: `ClassifyMVCCRewrite` (which MVCC versions a rotation must re-encrypt, iterating `(user_key, version_ts)` rather than `user_key`, so history under the retiring DEK is not left behind) and `RewriteThrottle` (the `--rate=N MiB/s` inter-batch yield, computed on cumulative bytes so a stalled job cannot bank idle time and burst). Decision layer only — no store, no clock, no Raft. | shipped | — | +| 9C+ | Rotation rewrap, the rewrite job's execution half, admission-control wiring, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production diff --git a/internal/encryption/rewrite_plan.go b/internal/encryption/rewrite_plan.go new file mode 100644 index 000000000..33f89bec5 --- /dev/null +++ b/internal/encryption/rewrite_plan.go @@ -0,0 +1,167 @@ +package encryption + +import "time" + +// §5.4 rewrite-job decision layer. +// +// The rewrite job re-encrypts data under a new DEK so an old one can be +// retired. §5.4 is emphatic that it is NOT a single-pass conversion of +// "the live value of every key": Pebble holds MVCC history, and the +// snapshot and lease-read paths can read back any version newer than +// minRetainedTS. A rewrite that touched only the live version would +// leave older versions under the retiring DEK and quietly break +// snapshot reads the moment it was unloaded. +// +// Two pure decisions live here — which versions to rewrite, and when to +// yield to stay inside the write-rate budget — so both are testable +// without a Pebble instance. Driving the iterator and the batch is the +// execution half. + +// RewriteVerdict is what the job does with one MVCC version. +type RewriteVerdict int + +const ( + // RewriteSkip leaves the version untouched. + RewriteSkip RewriteVerdict = iota + // RewriteReencrypt re-encrypts the value under the active DEK, at + // the SAME internal key. §5.4: no new MVCC version, no OCC + // conflict, no visible change to readers. + RewriteReencrypt +) + +func (v RewriteVerdict) String() string { + if v == RewriteReencrypt { + return "reencrypt" + } + return "skip" +} + +// Reasons, a closed set so the job can meter them directly. +const ( + RewriteReasonRetiringDEK = "retiring_dek" + RewriteReasonCleartext = "cleartext_migration" + RewriteReasonTombstone = "tombstone" + RewriteReasonAlreadyFresh = "already_under_active_dek" + RewriteReasonUnretained = "below_min_retained_ts" +) + +// RewriteDecision is a verdict plus its reason. +type RewriteDecision struct { + Verdict RewriteVerdict + Reason string +} + +// MVCCVersionRef is one retained version, as the iterator sees it. +// +// The iteration unit is (user_key, version_ts) rather than user_key — +// that distinction is the whole point of §5.4, so the type carries the +// timestamp rather than letting a caller forget it. +type MVCCVersionRef struct { + CommitTS uint64 + // KeyID is the DEK this version's envelope names, or zero when the + // version is stored cleartext. + KeyID uint32 + // Cleartext is the MVCC metadata bit saying this version predates + // the §7.1 cutover and holds no envelope. + Cleartext bool + // Tombstone versions carry no value bytes, so there is nothing to + // re-encrypt. + Tombstone bool + // ValueBytes sizes the write for the rate budget. + ValueBytes int64 +} + +// ClassifyMVCCRewrite decides one version's fate. +// +// retiringKeyID is the DEK being retired. migrateCleartext enables the +// §7.1 cleartext→encrypted sweep; it is separate from the retiring DEK +// because the two jobs run at different times and conflating them +// would have a routine rotation silently start encrypting data the +// operator had not opted in to encrypting. +// +// minRetainedTS is the MVCC retention floor: versions at or below it +// are unreachable by any snapshot or lease read, so rewriting them +// would be pure write amplification. +func ClassifyMVCCRewrite( + version MVCCVersionRef, retiringKeyID uint32, migrateCleartext bool, minRetainedTS uint64, +) RewriteDecision { + if version.Tombstone { + // No value bytes to re-encrypt. Checked first because a + // tombstone's KeyID is meaningless. + return RewriteDecision{Verdict: RewriteSkip, Reason: RewriteReasonTombstone} + } + if version.CommitTS <= minRetainedTS { + // Below the retention floor: no reader can reach it, and the + // retirement criterion (§5.4 item 4) is stated against + // minRetainedTS for exactly this reason. + return RewriteDecision{Verdict: RewriteSkip, Reason: RewriteReasonUnretained} + } + if version.Cleartext { + if migrateCleartext { + return RewriteDecision{Verdict: RewriteReencrypt, Reason: RewriteReasonCleartext} + } + return RewriteDecision{Verdict: RewriteSkip, Reason: RewriteReasonCleartext} + } + if version.KeyID == retiringKeyID { + return RewriteDecision{Verdict: RewriteReencrypt, Reason: RewriteReasonRetiringDEK} + } + return RewriteDecision{Verdict: RewriteSkip, Reason: RewriteReasonAlreadyFresh} +} + +// bytesPerMiB converts the operator-facing `--rate=N MiB/s` unit into +// the byte-denominated arithmetic the throttle runs on. +const bytesPerMiB = 1024 * 1024 + +// RewriteThrottle implements the §5.4 `--rate=N MiB/s` write-rate +// budget by telling the job how long to yield between batches. +// +// Rate-limiting the rewrite matters because it competes with live +// traffic for the same Pebble write path: an unthrottled sweep of MVCC +// history is a sustained write amplification spike against a database +// that is also serving requests. +type RewriteThrottle struct { + bytesPerSecond float64 + started time.Time + written int64 +} + +// NewRewriteThrottle returns a throttle for the given rate. A +// non-positive rate disables throttling, which is what --rate=0 means: +// run as fast as the store allows. +func NewRewriteThrottle(mibPerSecond float64, now time.Time) *RewriteThrottle { + return &RewriteThrottle{ + bytesPerSecond: mibPerSecond * bytesPerMiB, + started: now, + } +} + +// Record accounts for a committed batch and returns how long the job +// should yield before the next one. +// +// The delay is computed from CUMULATIVE bytes against cumulative +// elapsed time, not per batch. A per-batch calculation lets a job that +// stalls for other reasons "bank" idle time and then burst well above +// the configured rate — which is precisely the spike the budget exists +// to prevent. +func (t *RewriteThrottle) Record(batchBytes int64, now time.Time) time.Duration { + if t == nil || t.bytesPerSecond <= 0 { + return 0 + } + if batchBytes > 0 { + t.written += batchBytes + } + required := time.Duration(float64(t.written) / t.bytesPerSecond * float64(time.Second)) + elapsed := now.Sub(t.started) + if elapsed >= required { + return 0 + } + return required - elapsed +} + +// Written reports the cumulative bytes recorded. +func (t *RewriteThrottle) Written() int64 { + if t == nil { + return 0 + } + return t.written +} diff --git a/internal/encryption/rewrite_plan_test.go b/internal/encryption/rewrite_plan_test.go new file mode 100644 index 000000000..9e1ccef0a --- /dev/null +++ b/internal/encryption/rewrite_plan_test.go @@ -0,0 +1,216 @@ +package encryption_test + +import ( + "testing" + "time" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/stretchr/testify/require" +) + +const ( + retiringDEK = uint32(7) + activeDEK = uint32(8) + retainFloor = uint64(100) +) + +func version(over encryption.MVCCVersionRef) encryption.MVCCVersionRef { + if over.CommitTS == 0 { + over.CommitTS = retainFloor + 50 + } + return over +} + +// TestClassifyMVCCRewriteCoversEveryVersionShape is the §5.4 iteration +// contract. The unit is (user_key, version_ts): a rewrite that touched +// only live values would leave older versions under the retiring DEK +// and break snapshot reads the moment it was unloaded. +func TestClassifyMVCCRewriteCoversEveryVersionShape(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + version encryption.MVCCVersionRef + migrate bool + wantVerb encryption.RewriteVerdict + wantReason string + }{ + { + name: "version under the retiring DEK", + version: version(encryption.MVCCVersionRef{KeyID: retiringDEK}), + wantVerb: encryption.RewriteReencrypt, + wantReason: encryption.RewriteReasonRetiringDEK, + }, + { + name: "version already under the active DEK", + version: version(encryption.MVCCVersionRef{KeyID: activeDEK}), + wantVerb: encryption.RewriteSkip, + wantReason: encryption.RewriteReasonAlreadyFresh, + }, + { + name: "tombstone carries no value bytes", + version: version(encryption.MVCCVersionRef{KeyID: retiringDEK, Tombstone: true}), + wantVerb: encryption.RewriteSkip, + wantReason: encryption.RewriteReasonTombstone, + }, + { + name: "cleartext during a migration sweep", + version: version(encryption.MVCCVersionRef{Cleartext: true}), + migrate: true, + wantVerb: encryption.RewriteReencrypt, + wantReason: encryption.RewriteReasonCleartext, + }, + { + name: "cleartext outside a migration sweep", + version: version(encryption.MVCCVersionRef{Cleartext: true}), + migrate: false, + wantVerb: encryption.RewriteSkip, + wantReason: encryption.RewriteReasonCleartext, + }, + { + name: "version below the retention floor", + version: encryption.MVCCVersionRef{CommitTS: retainFloor - 1, KeyID: retiringDEK}, + wantVerb: encryption.RewriteSkip, + wantReason: encryption.RewriteReasonUnretained, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := encryption.ClassifyMVCCRewrite(tc.version, retiringDEK, tc.migrate, retainFloor) + require.Equal(t, tc.wantVerb, got.Verdict) + require.Equal(t, tc.wantReason, got.Reason) + }) + } +} + +// TestClassifyMVCCRewriteDoesNotEncryptCleartextDuringAPlainRotation +// keeps the two jobs separate. A routine rotation must not silently +// start encrypting data the operator never opted in to encrypting — +// that is the §7.1 migration, a different decision with a different +// blast radius. +func TestClassifyMVCCRewriteDoesNotEncryptCleartextDuringAPlainRotation(t *testing.T) { + t.Parallel() + + v := version(encryption.MVCCVersionRef{Cleartext: true}) + require.Equal(t, encryption.RewriteSkip, + encryption.ClassifyMVCCRewrite(v, retiringDEK, false, retainFloor).Verdict) + require.Equal(t, encryption.RewriteReencrypt, + encryption.ClassifyMVCCRewrite(v, retiringDEK, true, retainFloor).Verdict) +} + +// TestClassifyMVCCRewriteRewritesHistoricalVersionsNotJustTheLatest is +// the property §5.4 opens with: every RETAINED version under the +// retiring DEK must be rewritten, however old, or unloading the DEK +// breaks snapshot reads. +func TestClassifyMVCCRewriteRewritesHistoricalVersionsNotJustTheLatest(t *testing.T) { + t.Parallel() + + for _, ts := range []uint64{retainFloor + 1, retainFloor + 10, retainFloor + 10_000} { + got := encryption.ClassifyMVCCRewrite( + encryption.MVCCVersionRef{CommitTS: ts, KeyID: retiringDEK}, + retiringDEK, false, retainFloor) + require.Equal(t, encryption.RewriteReencrypt, got.Verdict, + "retained version at ts=%d must be rewritten", ts) + } +} + +// TestClassifyMVCCRewriteSkipsAtTheRetentionFloorBoundary pins the +// boundary against §5.4 item 4, which states retirement in terms of +// minRetainedTS: a version AT the floor is not reachable, so rewriting +// it is pure write amplification. +func TestClassifyMVCCRewriteSkipsAtTheRetentionFloorBoundary(t *testing.T) { + t.Parallel() + + at := encryption.ClassifyMVCCRewrite( + encryption.MVCCVersionRef{CommitTS: retainFloor, KeyID: retiringDEK}, + retiringDEK, false, retainFloor) + require.Equal(t, encryption.RewriteSkip, at.Verdict) + + above := encryption.ClassifyMVCCRewrite( + encryption.MVCCVersionRef{CommitTS: retainFloor + 1, KeyID: retiringDEK}, + retiringDEK, false, retainFloor) + require.Equal(t, encryption.RewriteReencrypt, above.Verdict) +} + +// --------------------------------------------------------------------------- +// Rate budget +// --------------------------------------------------------------------------- + +func TestRewriteThrottleYieldsToHoldTheConfiguredRate(t *testing.T) { + t.Parallel() + + start := time.Unix(1_700_000_000, 0) + // 1 MiB/s. + th := encryption.NewRewriteThrottle(1, start) + + // 2 MiB written instantly needs 2s of elapsed time. + require.Equal(t, 2*time.Second, th.Record(2*1024*1024, start)) + + // After 2s have actually passed, no further yield is owed. + require.Zero(t, th.Record(0, start.Add(2*time.Second))) +} + +// TestRewriteThrottleMeasuresCumulativeRateNotPerBatch pins the +// anti-burst property, and needs MULTIPLE batches to do it: with a +// single batch the cumulative and per-batch rules agree, so a one-batch +// test proves nothing. Successive batches at the same instant are what +// separate them — which is also the realistic burst shape, a job +// draining several batches back to back. +func TestRewriteThrottleMeasuresCumulativeRateNotPerBatch(t *testing.T) { + t.Parallel() + + start := time.Unix(1_700_000_000, 0) + // 1 MiB/s. + th := encryption.NewRewriteThrottle(1, start) + + // First MiB at t=0: owes 1s under either rule. + require.Equal(t, time.Second, th.Record(1024*1024, start)) + + // Second MiB, still at t=0. Cumulatively the job has now written + // 2 MiB and owes 2s against 0s elapsed. A per-batch rule would see + // only this batch's 1 MiB and owe 1s, letting the job sustain + // double the configured rate indefinitely. + require.Equal(t, 2*time.Second, th.Record(1024*1024, start), + "the debt must accumulate across batches, not reset with each one") + + // Third MiB after 2s have passed: 3 MiB owes 3s, 2s elapsed. + require.Equal(t, time.Second, + th.Record(1024*1024, start.Add(2*time.Second))) +} + +func TestRewriteThrottleDisabledAtNonPositiveRate(t *testing.T) { + t.Parallel() + + start := time.Unix(1_700_000_000, 0) + for _, rate := range []float64{0, -1} { + th := encryption.NewRewriteThrottle(rate, start) + require.Zero(t, th.Record(1<<30, start), "--rate=%v means run unthrottled", rate) + } +} + +func TestRewriteThrottleTracksCumulativeBytes(t *testing.T) { + t.Parallel() + + start := time.Unix(1_700_000_000, 0) + th := encryption.NewRewriteThrottle(1, start) + th.Record(1024, start) + th.Record(2048, start) + require.Equal(t, int64(3072), th.Written()) +} + +func TestRewriteThrottleNilReceiverIsInert(t *testing.T) { + t.Parallel() + + var th *encryption.RewriteThrottle + require.Zero(t, th.Record(1<<20, time.Unix(0, 0))) + require.Zero(t, th.Written()) +} + +func TestRewriteVerdictStringsAreStable(t *testing.T) { + t.Parallel() + + require.Equal(t, "reencrypt", encryption.RewriteReencrypt.String()) + require.Equal(t, "skip", encryption.RewriteSkip.String()) +} From b005252ade5f9c03766db353eb28fbfa18650950 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 11 Sep 2026 20:45:15 +0900 Subject: [PATCH 2/2] =?UTF-8?q?docs:=20keep=20the=20=C2=A79.2=20metrics=20?= =?UTF-8?q?named=20in=20the=20open=20encryption=20milestone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 9C-6 row had narrowed the open 9C+ scope to "remaining benchmarks and encrypted Jepsen", dropping the explicit mention of the §9.2 metrics. Neither elastickv_encryption_writes_per_dek{key_id} nor elastickv_encryption_last_proposed_index_per_raft_dek{key_id} exists outside this design document, so naming them keeps the gap visible rather than leaving it implied by the section reference alone. Same correction applied on #1233 and #1234, which narrowed the row the same way. Claude-Session: https://claude.ai/code/session_013rNHooj7NF3giihWVba8QE --- docs/design/2026_04_29_partial_data_at_rest_encryption.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 01dc17211..b2f251725 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -35,7 +35,7 @@ Date: 2026-04-29 | 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` | | 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | | 9C-6 | §5.4 rewrite-job decisions: `ClassifyMVCCRewrite` (which MVCC versions a rotation must re-encrypt, iterating `(user_key, version_ts)` rather than `user_key`, so history under the retiring DEK is not left behind) and `RewriteThrottle` (the `--rate=N MiB/s` inter-batch yield, computed on cumulative bytes so a stalled job cannot bank idle time and burst). Decision layer only — no store, no clock, no Raft. | shipped | — | -| 9C+ | Rotation rewrap, the rewrite job's execution half, admission-control wiring, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | +| 9C+ | Rotation rewrap, the rewrite job's execution half, admission-control wiring, the §9.2 metrics (including `elastickv_encryption_writes_per_dek{key_id}` and `elastickv_encryption_last_proposed_index_per_raft_dek{key_id}`, neither of which is exported yet), remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production