jepsen: run the existing suites against an encrypted cluster (§8.4)

docs/design/2026_04_29_partial_data_at_rest_encryption.md

@@ -2445,6 +2445,15 @@ different output bytes; FSM apply still deterministic), so no new
 Jepsen workload is required. A pass under the existing suite is the
 acceptance gate.
 
+**Implemented.** `lein test :only ...` aside, any workload accepts
+`--encryption`, which provisions the §5.1 KEK file on each node and
+starts the server with `--encryption-enabled`, `--kekFile` and
+`--encryptionSidecarPath`. The switch defaults off so the existing
+unencrypted runs are unchanged, and `server-args` is a pure function so
+a test can assert the flags actually reach the server — a `--encryption`
+run that silently produced an unencrypted cluster would report PASS and
+be recorded as evidence for this gate.
+
 ---
 
 ## 9. Operational concerns

jepsen/src/elastickv/cli.clj

@@ -14,6 +14,11 @@
     :default default-nodes-str]
    [nil "--local" "Run locally without SSH or nemesis."
     :default false]
+   ;; §8.4: the encrypted acceptance gate is the EXISTING suites run
+   ;; against an encrypted cluster, so this is a cluster-setup switch
+   ;; rather than a workload selector — every workload honours it.
+   [nil "--encryption" "Run against a cluster with data-at-rest encryption enabled."
+    :default false]
    [nil "--host HOST" "Host override for clients."
     :default nil]
    [nil "--grpc-port PORT" "gRPC/Raft port."

jepsen/src/elastickv/db.clj

@@ -15,6 +15,22 @@
 (def ^:private transport-metrics-file "/var/log/elastickv-transport-metrics.prom")
 (def ^:private pid-file "/var/run/elastickv.pid")
 (def ^:private server-bin (str bin-dir "/elastickv"))
+
+;; §8.4 acceptance gate: the encrypted-cluster run reuses the EXISTING
+;; Redis and DynamoDB workloads rather than adding a new one —
+;; encryption is consistency-transparent (same input bytes, different
+;; output bytes, apply still deterministic), so what has to be built is
+;; the ability to stand the cluster up encrypted, not a new checker.
+(def ^:private kek-file (str data-dir "/kek.bin"))
+(def ^:private sidecar-file (str data-dir "/keys.json"))
+
+;; kek-bytes is the §5.1 KEK: exactly 32 raw bytes, owner-only mode.
+;; A fixed test value, not a generated one — every node in the cluster
+;; must unwrap the same sidecar, and a per-node random KEK would make
+;; the cluster refuse to start with ErrKEKMismatch, which is a far more
+;; confusing failure than a hardcoded test key.
+(def ^:private kek-test-bytes
+  (apply str (repeat 32 "k")))
 (def ^:private raftadmin-bin (str bin-dir "/raftadmin"))
 
 (def ^:private build-dir
@@ -62,6 +78,19 @@
         (c/upload (str build-dir "/" bin) (str bin-dir "/" bin))
         (c/exec :chmod "755" (str bin-dir "/" bin))))))
 
+(defn- provision-kek!
+  "Writes the §5.1 KEK file with owner-only permissions.
+
+  Runs before start-node! because --encryption-enabled refuses to start
+  without a readable KEK source, and the refusal happens during startup
+  guards — well before anything the workload could observe."
+  [node]
+  (c/on node
+    (c/su
+      (c/exec :mkdir :-p data-dir)
+      (c/exec :bash :-c (str "printf '%s' '" kek-test-bytes "' > " kek-file))
+      (c/exec :chmod "600" kek-file))))
+
 (defn- node-addr
   "Returns host:port for the node and port."
   [node port]
@@ -103,8 +132,41 @@
 (defn- build-raft-dynamo-map [nodes grpc-port dynamo-port raft-groups]
   (build-raft-service-map nodes grpc-port dynamo-port raft-groups))
 
+(defn server-args
+  "Builds the elastickv server argv for one node.
+
+  Extracted from start-node! as a pure function so the flag set — and
+  in particular whether encryption is actually switched on — is
+  testable without SSH. A --encryption run that silently produced an
+  UNENCRYPTED cluster would report PASS and be recorded as evidence for
+  the §8.4 acceptance gate, which is worse than having no gate."
+  [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine
+           raft-redis-map raft-dynamo-map raft-groups shard-ranges
+           encryption bootstrap?]}]
+  (cond-> ["--address" grpc
+           "--redisAddress" redis
+           "--raftId" (name node)
+           "--raftDataDir" data-dir
+           "--raftEngine" (or raft-engine "etcd")
+           "--raftRedisMap" raft-redis-map]
+    dynamo (conj "--dynamoAddress" dynamo
+                 "--raftDynamoMap" raft-dynamo-map)
+    s3 (conj "--s3Address" s3)
+    sqs (conj "--sqsAddress" sqs)
+    (and sqs sqs-region) (conj "--sqsRegion" sqs-region)
+    (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups))
+    (seq shard-ranges) (conj "--shardRanges" shard-ranges)
+    ;; Sidecar path alone only enables read-only capability probing; the
+    ;; mutating RPCs the bootstrap needs also require
+    ;; --encryption-enabled AND a KEK source, so the three travel
+    ;; together or not at all.
+    encryption (conj "--encryptionSidecarPath" sidecar-file
+                     "--encryption-enabled"
+                     "--kekFile" kek-file)
+    bootstrap? (conj "--raftBootstrap")))
+
 (defn- start-node!
-  [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env]}]
+  [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env encryption]}]
   (when (and (seq raft-groups)
              (> (count raft-groups) 1)
              (nil? shard-ranges))
@@ -123,20 +185,12 @@
         raft-dynamo-map (when dynamo
                           (build-raft-dynamo-map (:nodes test) grpc-port dynamo-port raft-groups))
         bootstrap? (= node bootstrap-node)
-        args (cond-> ["--address" grpc
-                      "--redisAddress" redis
-                      "--raftId" (name node)
-                      "--raftDataDir" data-dir
-                      "--raftEngine" (or raft-engine "etcd")
-                      "--raftRedisMap" raft-redis-map]
-               dynamo (conj "--dynamoAddress" dynamo
-                            "--raftDynamoMap" raft-dynamo-map)
-               s3 (conj "--s3Address" s3)
-               sqs (conj "--sqsAddress" sqs)
-               (and sqs sqs-region) (conj "--sqsRegion" sqs-region)
-               (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups))
-               (seq shard-ranges) (conj "--shardRanges" shard-ranges)
-               bootstrap? (conj "--raftBootstrap"))
+        args (server-args
+               {:node node :grpc grpc :redis redis :dynamo dynamo :s3 s3 :sqs sqs
+                :sqs-region sqs-region :data-dir data-dir :raft-engine raft-engine
+                :raft-redis-map raft-redis-map :raft-dynamo-map raft-dynamo-map
+                :raft-groups raft-groups :shard-ranges shard-ranges
+                :encryption encryption :bootstrap? bootstrap?})
         daemon-opts (cond-> {:chdir bin-dir
                              :logfile log-file
                              :pidfile pid-file
@@ -194,6 +248,8 @@
       (c/su
         (c/exec :mkdir :-p data-dir)
         (c/exec :rm :-f log-file transport-metrics-file)))
+    (when (:encryption opts)
+      (provision-kek! node))
     (start-node! test node (merge {:data-dir data-dir
                                    :grpc-port (or (:grpc-port opts) 50051)
                                    :redis-port (or (:redis-port opts) 6379)

jepsen/src/elastickv/dynamodb_workload.clj

@@ -329,7 +329,8 @@
                                   :redis-port   (or (:redis-port opts) 6379)
                                   :dynamo-port  node->port
                                   :raft-groups  (:raft-groups opts)
-                                  :shard-ranges (:shard-ranges opts)}))
+                                  :shard-ranges (:shard-ranges opts)
+                                  :encryption   (:encryption opts)}))
          rate         (double (or (:rate opts) 5))
          time-limit   (or (:time-limit opts) 30)
          faults       (if local?

jepsen/src/elastickv/redis_workload.clj

@@ -125,7 +125,8 @@
                       (ekdb/db {:grpc-port (or (:grpc-port opts) 50051)
                                 :redis-port node->port
                                 :raft-groups (:raft-groups opts)
-                                :shard-ranges (:shard-ranges opts)}))
+                                :shard-ranges (:shard-ranges opts)
+                                :encryption (:encryption opts)}))
          rate       (double (or (:rate opts) 5))
          time-limit (or (:time-limit opts) 30)
          faults     (if local?

jepsen/test/elastickv/encrypted_cluster_test.clj

@@ -0,0 +1,86 @@
+(ns elastickv.encrypted-cluster-test
+  "Pins the §8.4 encrypted acceptance gate: the EXISTING Redis and
+  DynamoDB workloads must be runnable against a cluster with
+  data-at-rest encryption enabled.
+
+  The design is explicit that no new workload is required — encryption
+  is consistency-transparent, so the gate is a cluster-setup switch. The
+  risk that switch carries is that it silently does nothing: a run that
+  reported PASS while the cluster was never actually encrypted would be
+  worse than no gate at all, because it would be recorded as evidence."
+  (:require [clojure.test :refer :all]
+            [elastickv.cli :as cli]
+            [elastickv.db :as ekdb]
+            [elastickv.dynamodb-workload :as dynamo]
+            [elastickv.redis-workload :as redis]))
+
+(deftest encryption-flag-is-available-to-every-workload
+  ;; It lives in common-cli-opts rather than per-workload, so a future
+  ;; workload gets the gate without opting in.
+  (let [names (set (map second cli/common-cli-opts))]
+    (is (contains? names "--encryption"))))
+
+(deftest encryption-defaults-off
+  ;; The unencrypted suites are the existing baseline; turning this on
+  ;; by default would silently change what every current run measures.
+  (let [spec (first (filter #(= "--encryption" (second %)) cli/common-cli-opts))]
+    (is (false? (:default (apply hash-map (drop 3 spec)))))))
+
+(deftest redis-workload-propagates-encryption-to-the-db
+  (let [test-map (redis/elastickv-redis-test {:encryption true})]
+    (is (true? (get-in test-map [:db :opts :encryption])))))
+
+(deftest dynamodb-workload-propagates-encryption-to-the-db
+  (let [test-map (dynamo/elastickv-dynamodb-test {:encryption true})]
+    (is (true? (get-in test-map [:db :opts :encryption])))))
+
+(deftest encryption-is-absent-from-the-db-when-not-requested
+  ;; The flag must not leak a truthy value into an ordinary run.
+  (let [test-map (redis/elastickv-redis-test {})]
+    (is (not (true? (get-in test-map [:db :opts :encryption]))))))
+
+(deftest db-accepts-the-encryption-option
+  ;; ekdb/db carries opts verbatim; this pins that the key survives
+  ;; construction rather than being dropped by a destructuring form.
+  (let [db (ekdb/db {:grpc-port 50051 :encryption true})]
+    (is (true? (get-in db [:opts :encryption])))))
+
+;; ---------------------------------------------------------------------------
+;; The load-bearing property: the switch must actually reach the server
+;; ---------------------------------------------------------------------------
+
+(defn- args-for [over]
+  (ekdb/server-args (merge {:node "n1" :grpc "n1:50051" :redis "n1:6379"
+                            :data-dir "/var/lib/elastickv"
+                            :raft-redis-map "n1=n1:6379"}
+                           over)))
+
+(deftest encryption-emits-all-three-server-flags
+  ;; A sidecar path alone only enables read-only capability probing.
+  ;; The mutating RPCs the bootstrap needs require --encryption-enabled
+  ;; AND a KEK source, so all three must travel together — two of the
+  ;; three would produce a cluster that refuses to start, or worse, one
+  ;; that starts unencrypted.
+  (let [args (set (args-for {:encryption true}))]
+    (is (contains? args "--encryption-enabled"))
+    (is (contains? args "--encryptionSidecarPath"))
+    (is (contains? args "--kekFile"))))
+
+(deftest without-encryption-no-encryption-flag-is-emitted
+  ;; The unencrypted suites must be byte-identical to before, or the
+  ;; baseline every existing run measures has silently changed.
+  (let [args (set (args-for {}))]
+    (is (not (contains? args "--encryption-enabled")))
+    (is (not (contains? args "--encryptionSidecarPath")))
+    (is (not (contains? args "--kekFile")))))
+
+(deftest encryption-does-not-disturb-the-other-flags
+  (let [plain     (remove #{"--encryptionSidecarPath" "--encryption-enabled" "--kekFile"}
+                          (args-for {:encryption true}))
+        encrypted (args-for {})]
+    ;; Removing the encryption flags and their values must leave the
+    ;; same argv an unencrypted run would produce.
+    (is (= (set encrypted)
+           (set (remove #(or (= % "/var/lib/elastickv/keys.json")
+                             (= % "/var/lib/elastickv/kek.bin"))
+                        plain))))))