From 395f7d625995ef88797a8f13f2828a75259266d5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 11 Sep 2026 00:03:37 +0900 Subject: [PATCH] =?UTF-8?q?jepsen:=20run=20the=20existing=20suites=20again?= =?UTF-8?q?st=20an=20encrypted=20cluster=20(=C2=A78.4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encryption design's Jepsen gate is explicit that NO new workload is required: encryption is consistency-transparent, so the acceptance gate is the existing Redis and DynamoDB workloads passing against an encrypted cluster. What was missing was the ability to stand that cluster up. --encryption provisions the §5.1 KEK file on each node with owner-only permissions and starts the server with --encryption-enabled, --kekFile and --encryptionSidecarPath. The three travel together because a sidecar path alone only enables read-only capability probing, while the mutating RPCs a bootstrap needs require the flag and a KEK source too. The KEK is a fixed test value rather than a generated one: every node must unwrap the same sidecar, and a per-node random KEK would fail startup with ErrKEKMismatch — a far more confusing failure than a hardcoded test key. The flag lives in common-cli-opts, so a future workload inherits the gate without opting in, and defaults off so existing runs measure the same thing they did before. server-args is extracted from start-node! as a pure function purely so the flag set is testable without SSH. That matters more than it looks: a --encryption run that silently produced an UNENCRYPTED cluster would report PASS and be recorded as evidence for this gate, which is worse than having no gate. The first version of these tests did not pin it — dropping the flags from db.clj left them all green. Full suite: 157 tests, 0 failures. Claude-Session: https://claude.ai/code/session_013rNHooj7NF3giihWVba8QE --- ...6_04_29_partial_data_at_rest_encryption.md | 9 ++ jepsen/src/elastickv/cli.clj | 5 ++ jepsen/src/elastickv/db.clj | 86 +++++++++++++++---- jepsen/src/elastickv/dynamodb_workload.clj | 3 +- jepsen/src/elastickv/redis_workload.clj | 3 +- .../test/elastickv/encrypted_cluster_test.clj | 86 +++++++++++++++++++ 6 files changed, 175 insertions(+), 17 deletions(-) create mode 100644 jepsen/test/elastickv/encrypted_cluster_test.clj diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0588b2805..55afe3ce0 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -2445,6 +2445,15 @@ different output bytes; FSM apply still deterministic), so no new Jepsen workload is required. A pass under the existing suite is the acceptance gate. +**Implemented.** `lein test :only ...` aside, any workload accepts +`--encryption`, which provisions the §5.1 KEK file on each node and +starts the server with `--encryption-enabled`, `--kekFile` and +`--encryptionSidecarPath`. The switch defaults off so the existing +unencrypted runs are unchanged, and `server-args` is a pure function so +a test can assert the flags actually reach the server — a `--encryption` +run that silently produced an unencrypted cluster would report PASS and +be recorded as evidence for this gate. + --- ## 9. Operational concerns diff --git a/jepsen/src/elastickv/cli.clj b/jepsen/src/elastickv/cli.clj index a802e9ab0..95cc2883c 100644 --- a/jepsen/src/elastickv/cli.clj +++ b/jepsen/src/elastickv/cli.clj @@ -14,6 +14,11 @@ :default default-nodes-str] [nil "--local" "Run locally without SSH or nemesis." :default false] + ;; §8.4: the encrypted acceptance gate is the EXISTING suites run + ;; against an encrypted cluster, so this is a cluster-setup switch + ;; rather than a workload selector — every workload honours it. + [nil "--encryption" "Run against a cluster with data-at-rest encryption enabled." + :default false] [nil "--host HOST" "Host override for clients." :default nil] [nil "--grpc-port PORT" "gRPC/Raft port." diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 369d33bc9..e804bcc73 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -15,6 +15,22 @@ (def ^:private transport-metrics-file "/var/log/elastickv-transport-metrics.prom") (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) + +;; §8.4 acceptance gate: the encrypted-cluster run reuses the EXISTING +;; Redis and DynamoDB workloads rather than adding a new one — +;; encryption is consistency-transparent (same input bytes, different +;; output bytes, apply still deterministic), so what has to be built is +;; the ability to stand the cluster up encrypted, not a new checker. +(def ^:private kek-file (str data-dir "/kek.bin")) +(def ^:private sidecar-file (str data-dir "/keys.json")) + +;; kek-bytes is the §5.1 KEK: exactly 32 raw bytes, owner-only mode. +;; A fixed test value, not a generated one — every node in the cluster +;; must unwrap the same sidecar, and a per-node random KEK would make +;; the cluster refuse to start with ErrKEKMismatch, which is a far more +;; confusing failure than a hardcoded test key. +(def ^:private kek-test-bytes + (apply str (repeat 32 "k"))) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) (def ^:private build-dir @@ -62,6 +78,19 @@ (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) +(defn- provision-kek! + "Writes the §5.1 KEK file with owner-only permissions. + + Runs before start-node! because --encryption-enabled refuses to start + without a readable KEK source, and the refusal happens during startup + guards — well before anything the workload could observe." + [node] + (c/on node + (c/su + (c/exec :mkdir :-p data-dir) + (c/exec :bash :-c (str "printf '%s' '" kek-test-bytes "' > " kek-file)) + (c/exec :chmod "600" kek-file)))) + (defn- node-addr "Returns host:port for the node and port." [node port] @@ -103,8 +132,41 @@ (defn- build-raft-dynamo-map [nodes grpc-port dynamo-port raft-groups] (build-raft-service-map nodes grpc-port dynamo-port raft-groups)) +(defn server-args + "Builds the elastickv server argv for one node. + + Extracted from start-node! as a pure function so the flag set — and + in particular whether encryption is actually switched on — is + testable without SSH. A --encryption run that silently produced an + UNENCRYPTED cluster would report PASS and be recorded as evidence for + the §8.4 acceptance gate, which is worse than having no gate." + [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine + raft-redis-map raft-dynamo-map raft-groups shard-ranges + encryption bootstrap?]}] + (cond-> ["--address" grpc + "--redisAddress" redis + "--raftId" (name node) + "--raftDataDir" data-dir + "--raftEngine" (or raft-engine "etcd") + "--raftRedisMap" raft-redis-map] + dynamo (conj "--dynamoAddress" dynamo + "--raftDynamoMap" raft-dynamo-map) + s3 (conj "--s3Address" s3) + sqs (conj "--sqsAddress" sqs) + (and sqs sqs-region) (conj "--sqsRegion" sqs-region) + (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) + (seq shard-ranges) (conj "--shardRanges" shard-ranges) + ;; Sidecar path alone only enables read-only capability probing; the + ;; mutating RPCs the bootstrap needs also require + ;; --encryption-enabled AND a KEK source, so the three travel + ;; together or not at all. + encryption (conj "--encryptionSidecarPath" sidecar-file + "--encryption-enabled" + "--kekFile" kek-file) + bootstrap? (conj "--raftBootstrap"))) + (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env encryption]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -123,20 +185,12 @@ raft-dynamo-map (when dynamo (build-raft-dynamo-map (:nodes test) grpc-port dynamo-port raft-groups)) bootstrap? (= node bootstrap-node) - args (cond-> ["--address" grpc - "--redisAddress" redis - "--raftId" (name node) - "--raftDataDir" data-dir - "--raftEngine" (or raft-engine "etcd") - "--raftRedisMap" raft-redis-map] - dynamo (conj "--dynamoAddress" dynamo - "--raftDynamoMap" raft-dynamo-map) - s3 (conj "--s3Address" s3) - sqs (conj "--sqsAddress" sqs) - (and sqs sqs-region) (conj "--sqsRegion" sqs-region) - (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) - (seq shard-ranges) (conj "--shardRanges" shard-ranges) - bootstrap? (conj "--raftBootstrap")) + args (server-args + {:node node :grpc grpc :redis redis :dynamo dynamo :s3 s3 :sqs sqs + :sqs-region sqs-region :data-dir data-dir :raft-engine raft-engine + :raft-redis-map raft-redis-map :raft-dynamo-map raft-dynamo-map + :raft-groups raft-groups :shard-ranges shard-ranges + :encryption encryption :bootstrap? bootstrap?}) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file @@ -194,6 +248,8 @@ (c/su (c/exec :mkdir :-p data-dir) (c/exec :rm :-f log-file transport-metrics-file))) + (when (:encryption opts) + (provision-kek! node)) (start-node! test node (merge {:data-dir data-dir :grpc-port (or (:grpc-port opts) 50051) :redis-port (or (:redis-port opts) 6379) diff --git a/jepsen/src/elastickv/dynamodb_workload.clj b/jepsen/src/elastickv/dynamodb_workload.clj index c5d586a43..87a0f7ad8 100644 --- a/jepsen/src/elastickv/dynamodb_workload.clj +++ b/jepsen/src/elastickv/dynamodb_workload.clj @@ -329,7 +329,8 @@ :redis-port (or (:redis-port opts) 6379) :dynamo-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/redis_workload.clj b/jepsen/src/elastickv/redis_workload.clj index 20283a64c..ac4f5d3f2 100644 --- a/jepsen/src/elastickv/redis_workload.clj +++ b/jepsen/src/elastickv/redis_workload.clj @@ -125,7 +125,8 @@ (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) :redis-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/test/elastickv/encrypted_cluster_test.clj b/jepsen/test/elastickv/encrypted_cluster_test.clj new file mode 100644 index 000000000..6e9d4585b --- /dev/null +++ b/jepsen/test/elastickv/encrypted_cluster_test.clj @@ -0,0 +1,86 @@ +(ns elastickv.encrypted-cluster-test + "Pins the §8.4 encrypted acceptance gate: the EXISTING Redis and + DynamoDB workloads must be runnable against a cluster with + data-at-rest encryption enabled. + + The design is explicit that no new workload is required — encryption + is consistency-transparent, so the gate is a cluster-setup switch. The + risk that switch carries is that it silently does nothing: a run that + reported PASS while the cluster was never actually encrypted would be + worse than no gate at all, because it would be recorded as evidence." + (:require [clojure.test :refer :all] + [elastickv.cli :as cli] + [elastickv.db :as ekdb] + [elastickv.dynamodb-workload :as dynamo] + [elastickv.redis-workload :as redis])) + +(deftest encryption-flag-is-available-to-every-workload + ;; It lives in common-cli-opts rather than per-workload, so a future + ;; workload gets the gate without opting in. + (let [names (set (map second cli/common-cli-opts))] + (is (contains? names "--encryption")))) + +(deftest encryption-defaults-off + ;; The unencrypted suites are the existing baseline; turning this on + ;; by default would silently change what every current run measures. + (let [spec (first (filter #(= "--encryption" (second %)) cli/common-cli-opts))] + (is (false? (:default (apply hash-map (drop 3 spec))))))) + +(deftest redis-workload-propagates-encryption-to-the-db + (let [test-map (redis/elastickv-redis-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest dynamodb-workload-propagates-encryption-to-the-db + (let [test-map (dynamo/elastickv-dynamodb-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest encryption-is-absent-from-the-db-when-not-requested + ;; The flag must not leak a truthy value into an ordinary run. + (let [test-map (redis/elastickv-redis-test {})] + (is (not (true? (get-in test-map [:db :opts :encryption])))))) + +(deftest db-accepts-the-encryption-option + ;; ekdb/db carries opts verbatim; this pins that the key survives + ;; construction rather than being dropped by a destructuring form. + (let [db (ekdb/db {:grpc-port 50051 :encryption true})] + (is (true? (get-in db [:opts :encryption]))))) + +;; --------------------------------------------------------------------------- +;; The load-bearing property: the switch must actually reach the server +;; --------------------------------------------------------------------------- + +(defn- args-for [over] + (ekdb/server-args (merge {:node "n1" :grpc "n1:50051" :redis "n1:6379" + :data-dir "/var/lib/elastickv" + :raft-redis-map "n1=n1:6379"} + over))) + +(deftest encryption-emits-all-three-server-flags + ;; A sidecar path alone only enables read-only capability probing. + ;; The mutating RPCs the bootstrap needs require --encryption-enabled + ;; AND a KEK source, so all three must travel together — two of the + ;; three would produce a cluster that refuses to start, or worse, one + ;; that starts unencrypted. + (let [args (set (args-for {:encryption true}))] + (is (contains? args "--encryption-enabled")) + (is (contains? args "--encryptionSidecarPath")) + (is (contains? args "--kekFile")))) + +(deftest without-encryption-no-encryption-flag-is-emitted + ;; The unencrypted suites must be byte-identical to before, or the + ;; baseline every existing run measures has silently changed. + (let [args (set (args-for {}))] + (is (not (contains? args "--encryption-enabled"))) + (is (not (contains? args "--encryptionSidecarPath"))) + (is (not (contains? args "--kekFile"))))) + +(deftest encryption-does-not-disturb-the-other-flags + (let [plain (remove #{"--encryptionSidecarPath" "--encryption-enabled" "--kekFile"} + (args-for {:encryption true})) + encrypted (args-for {})] + ;; Removing the encryption flags and their values must leave the + ;; same argv an unencrypted run would produce. + (is (= (set encrypted) + (set (remove #(or (= % "/var/lib/elastickv/keys.json") + (= % "/var/lib/elastickv/kek.bin")) + plain))))))