diff --git a/cmd/jepsen-encryption-setup/main.go b/cmd/jepsen-encryption-setup/main.go new file mode 100644 index 000000000..b279b0c41 --- /dev/null +++ b/cmd/jepsen-encryption-setup/main.go @@ -0,0 +1,72 @@ +// Command jepsen-encryption-setup produces a KEK-wrapped DEK for the Jepsen +// harness, which needs one to call `elastickv-admin encryption bootstrap`. +// +// This is TEST HARNESS TOOLING, not an operator tool. It exists because +// bootstrap takes the wrapped DEK bytes as an argument -- an operator gets them +// from their KMS -- and the Jepsen harness has only a local KEK file. Keeping it +// here rather than adding an `elastickv-admin` subcommand avoids putting DEK +// generation into operator-facing tooling, where handling raw key material +// needs its own design and review. +// +// The plaintext DEK never leaves this process: it is generated, wrapped under +// the KEK, and only the wrapped form is printed. The wrapped DEK is safe to pass +// on a command line, which is the whole point of the envelope scheme. +package main + +import ( + "crypto/rand" + "encoding/base64" + "flag" + "fmt" + "os" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/cockroachdb/errors" +) + +func main() { + if err := run(os.Args[1:], os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "jepsen-encryption-setup: %v\n", err) + os.Exit(1) + } +} + +func run(args []string, out *os.File) error { + fs := flag.NewFlagSet("jepsen-encryption-setup", flag.ContinueOnError) + kekFile := fs.String("kek-file", "", "path to the §5.1 KEK file (32 raw bytes, owner-only mode)") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return errors.Wrap(err, "parse flags") + } + if *kekFile == "" { + return errors.New("--kek-file is required") + } + wrapped, err := wrapFreshDEK(*kekFile) + if err != nil { + return err + } + if _, err := fmt.Fprintln(out, wrapped); err != nil { + return errors.Wrap(err, "write wrapped dek") + } + return nil +} + +// wrapFreshDEK generates one AES-256 DEK and returns its base64 wrapped form. +func wrapFreshDEK(kekFile string) (string, error) { + wrapper, err := kek.NewFileWrapper(kekFile) + if err != nil { + return "", errors.Wrap(err, "open kek file") + } + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + return "", errors.Wrap(err, "generate dek") + } + wrapped, err := wrapper.Wrap(dek) + if err != nil { + return "", errors.Wrap(err, "wrap dek") + } + return base64.StdEncoding.EncodeToString(wrapped), nil +} diff --git a/cmd/jepsen-encryption-setup/main_test.go b/cmd/jepsen-encryption-setup/main_test.go new file mode 100644 index 000000000..ed3fd2e46 --- /dev/null +++ b/cmd/jepsen-encryption-setup/main_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/stretchr/testify/require" +) + +func writeTestKEK(t *testing.T) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "kek") + kekBytes := make([]byte, encryption.KeySize) + for i := range kekBytes { + kekBytes[i] = byte(i + 1) + } + require.NoError(t, os.WriteFile(path, kekBytes, 0o600)) + return path +} + +// The wrapped DEK must round-trip through the same wrapper the server uses, or +// bootstrap would be handed bytes the node cannot unwrap -- and the Jepsen suite +// would fail at bootstrap instead of running encrypted. +func TestWrapFreshDEKProducesAnUnwrappableDEK(t *testing.T) { + t.Parallel() + + path := writeTestKEK(t) + encoded, err := wrapFreshDEK(path) + require.NoError(t, err) + + raw, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + wrapper, err := kek.NewFileWrapper(path) + require.NoError(t, err) + dek, err := wrapper.Unwrap(raw) + require.NoError(t, err) + require.Len(t, dek, encryption.KeySize, + "bootstrap rejects a DEK that is not AES-256") +} + +// Two invocations must not produce the same DEK: the harness calls this once for +// the storage DEK and once for the raft DEK, and bootstrap requires them to +// differ. +func TestWrapFreshDEKGeneratesADistinctDEKEachCall(t *testing.T) { + t.Parallel() + + path := writeTestKEK(t) + wrapper, err := kek.NewFileWrapper(path) + require.NoError(t, err) + + seen := make(map[string]struct{}, 8) + for range 8 { + encoded, err := wrapFreshDEK(path) + require.NoError(t, err) + raw, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + dek, err := wrapper.Unwrap(raw) + require.NoError(t, err) + _, dup := seen[string(dek)] + require.False(t, dup, "each call must generate a fresh DEK") + seen[string(dek)] = struct{}{} + } +} + +func TestRunRequiresAKEKFile(t *testing.T) { + t.Parallel() + + require.Error(t, run(nil, os.Stdout)) +} diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0588b2805..55afe3ce0 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -2445,6 +2445,15 @@ different output bytes; FSM apply still deterministic), so no new Jepsen workload is required. A pass under the existing suite is the acceptance gate. +**Implemented.** `lein test :only ...` aside, any workload accepts +`--encryption`, which provisions the §5.1 KEK file on each node and +starts the server with `--encryption-enabled`, `--kekFile` and +`--encryptionSidecarPath`. The switch defaults off so the existing +unencrypted runs are unchanged, and `server-args` is a pure function so +a test can assert the flags actually reach the server — a `--encryption` +run that silently produced an unencrypted cluster would report PASS and +be recorded as evidence for this gate. + --- ## 9. Operational concerns diff --git a/jepsen/src/elastickv/cli.clj b/jepsen/src/elastickv/cli.clj index a802e9ab0..95cc2883c 100644 --- a/jepsen/src/elastickv/cli.clj +++ b/jepsen/src/elastickv/cli.clj @@ -14,6 +14,11 @@ :default default-nodes-str] [nil "--local" "Run locally without SSH or nemesis." :default false] + ;; §8.4: the encrypted acceptance gate is the EXISTING suites run + ;; against an encrypted cluster, so this is a cluster-setup switch + ;; rather than a workload selector — every workload honours it. + [nil "--encryption" "Run against a cluster with data-at-rest encryption enabled." + :default false] [nil "--host HOST" "Host override for clients." :default nil] [nil "--grpc-port PORT" "gRPC/Raft port." diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 369d33bc9..e0c7aeac4 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -15,7 +15,30 @@ (def ^:private transport-metrics-file "/var/log/elastickv-transport-metrics.prom") (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) + +;; §8.4 acceptance gate: the encrypted-cluster run reuses the EXISTING +;; Redis and DynamoDB workloads rather than adding a new one — +;; encryption is consistency-transparent (same input bytes, different +;; output bytes, apply still deterministic), so what has to be built is +;; the ability to stand the cluster up encrypted, not a new checker. +(def ^:private kek-file (str data-dir "/kek.bin")) +(def ^:private sidecar-file (str data-dir "/keys.json")) + +;; kek-bytes is the §5.1 KEK: exactly 32 raw bytes, owner-only mode. +;; A fixed test value, not a generated one — every node in the cluster +;; must unwrap the same sidecar, and a per-node random KEK would make +;; the cluster refuse to start with ErrKEKMismatch, which is a far more +;; confusing failure than a hardcoded test key. +(def ^:private kek-test-bytes + (apply str (repeat 32 "k"))) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) +(def ^:private admin-bin (str bin-dir "/elastickv-admin")) +(def ^:private encryption-setup-bin (str bin-dir "/jepsen-encryption-setup")) + +;; §5.2 DEK ids for the bootstrap. Any non-zero pair that differs is valid; +;; bootstrap rejects zero and rejects the two being equal. +(def ^:private storage-dek-id 1) +(def ^:private raft-dek-id 2) (def ^:private build-dir ;; local (control node) directory for built binaries @@ -38,7 +61,9 @@ "GOPATH" "/home/vagrant/go" "GOCACHE" "/home/vagrant/.cache/go-build"})] (doseq [[out-cmd args] [["elastickv" ["go" "build" "-o" (str build-dir "/elastickv") "./cmd/server"]] - ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]]]] + ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]] + ["elastickv-admin" ["go" "build" "-o" (str build-dir "/elastickv-admin") "./cmd/elastickv-admin"]] + ["jepsen-encryption-setup" ["go" "build" "-o" (str build-dir "/jepsen-encryption-setup") "./cmd/jepsen-encryption-setup"]]]] (let [{:keys [exit err]} (apply sh/sh (concat args [:env env :dir root]))] (when-not (zero? exit) (throw (ex-info (str "failed to build " out-cmd) {:err err}))))))) @@ -58,10 +83,23 @@ (c/on node (c/su (c/exec :mkdir :-p bin-dir) - (doseq [bin ["elastickv" "raftadmin"]] + (doseq [bin ["elastickv" "raftadmin" "elastickv-admin" "jepsen-encryption-setup"]] (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) +(defn- provision-kek! + "Writes the §5.1 KEK file with owner-only permissions. + + Runs before start-node! because --encryption-enabled refuses to start + without a readable KEK source, and the refusal happens during startup + guards — well before anything the workload could observe." + [node] + (c/on node + (c/su + (c/exec :mkdir :-p data-dir) + (c/exec :bash :-c (str "printf '%s' '" kek-test-bytes "' > " kek-file)) + (c/exec :chmod "600" kek-file)))) + (defn- node-addr "Returns host:port for the node and port." [node port] @@ -103,8 +141,41 @@ (defn- build-raft-dynamo-map [nodes grpc-port dynamo-port raft-groups] (build-raft-service-map nodes grpc-port dynamo-port raft-groups)) +(defn server-args + "Builds the elastickv server argv for one node. + + Extracted from start-node! as a pure function so the flag set — and + in particular whether encryption is actually switched on — is + testable without SSH. A --encryption run that silently produced an + UNENCRYPTED cluster would report PASS and be recorded as evidence for + the §8.4 acceptance gate, which is worse than having no gate." + [{:keys [node grpc redis dynamo s3 sqs sqs-region data-dir raft-engine + raft-redis-map raft-dynamo-map raft-groups shard-ranges + encryption bootstrap?]}] + (cond-> ["--address" grpc + "--redisAddress" redis + "--raftId" (name node) + "--raftDataDir" data-dir + "--raftEngine" (or raft-engine "etcd") + "--raftRedisMap" raft-redis-map] + dynamo (conj "--dynamoAddress" dynamo + "--raftDynamoMap" raft-dynamo-map) + s3 (conj "--s3Address" s3) + sqs (conj "--sqsAddress" sqs) + (and sqs sqs-region) (conj "--sqsRegion" sqs-region) + (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) + (seq shard-ranges) (conj "--shardRanges" shard-ranges) + ;; Sidecar path alone only enables read-only capability probing; the + ;; mutating RPCs the bootstrap needs also require + ;; --encryption-enabled AND a KEK source, so the three travel + ;; together or not at all. + encryption (conj "--encryptionSidecarPath" sidecar-file + "--encryption-enabled" + "--kekFile" kek-file) + bootstrap? (conj "--raftBootstrap"))) + (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env encryption]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -123,20 +194,12 @@ raft-dynamo-map (when dynamo (build-raft-dynamo-map (:nodes test) grpc-port dynamo-port raft-groups)) bootstrap? (= node bootstrap-node) - args (cond-> ["--address" grpc - "--redisAddress" redis - "--raftId" (name node) - "--raftDataDir" data-dir - "--raftEngine" (or raft-engine "etcd") - "--raftRedisMap" raft-redis-map] - dynamo (conj "--dynamoAddress" dynamo - "--raftDynamoMap" raft-dynamo-map) - s3 (conj "--s3Address" s3) - sqs (conj "--sqsAddress" sqs) - (and sqs sqs-region) (conj "--sqsRegion" sqs-region) - (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) - (seq shard-ranges) (conj "--shardRanges" shard-ranges) - bootstrap? (conj "--raftBootstrap")) + args (server-args + {:node node :grpc grpc :redis redis :dynamo dynamo :s3 s3 :sqs sqs + :sqs-region sqs-region :data-dir data-dir :raft-engine raft-engine + :raft-redis-map raft-redis-map :raft-dynamo-map raft-dynamo-map + :raft-groups raft-groups :shard-ranges shard-ranges + :encryption encryption :bootstrap? bootstrap?}) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file @@ -185,6 +248,110 @@ (c/exec :env "RAFTADMIN_ALLOW_INSECURE=true" raftadmin-bin leader-addr "add_voter" peer-id peer-addr "0")))) +(defn encryption-endpoint + "Returns the gRPC address the EncryptionAdmin RPCs must be sent to. + + The bootstrap and cutover entries are proposed through the DEFAULT Raft + group, so a multi-group deployment must be addressed on that group's port + rather than on whichever port happens to be first in the map. group-ids is + sorted, so the lowest group id is the default one." + [node grpc-port raft-groups] + (if (seq raft-groups) + (group-addr node raft-groups (first (group-ids raft-groups))) + (node-addr node grpc-port))) + +(defn bootstrap-args + "argv for `elastickv-admin encryption bootstrap`. + + The writer batch comes from --discover-from rather than hand-written + --writer entries: §5.6 step 1a requires one registry entry per member, and + polling GetCapability is the only way to learn each node's real full_node_id + and local_epoch. A hand-written batch would go stale the moment a node + restarted and bumped its epoch." + [endpoint peer-endpoints wrapped-storage wrapped-raft] + (concat [admin-bin "encryption" "bootstrap" + (str "--endpoint=" endpoint) + (str "--storage-dek-id=" storage-dek-id) + (str "--raft-dek-id=" raft-dek-id) + (str "--wrapped-storage-dek=" wrapped-storage) + (str "--wrapped-raft-dek=" wrapped-raft)] + (map #(str "--discover-from=" %) peer-endpoints))) + +(defn enable-storage-envelope-args + "argv for `elastickv-admin encryption enable-storage-envelope`. + + This is the step that actually makes writes ciphertext: --encryption-enabled + only opens the mutator RPCs, and buildEncryptionWriteWiring keeps the store + gate closed until this entry applies." + [endpoint full-node-id local-epoch] + [admin-bin "encryption" "enable-storage-envelope" + (str "--endpoint=" endpoint) + (str "--proposer-node-id=" full-node-id) + (str "--proposer-local-epoch=" local-epoch)]) + +(defn parse-encryption-status + "Parses `elastickv-admin encryption status` output into a map. + + Returns :full-node-id and :local-epoch (needed as the proposer identity for + the cutover) and :storage-envelope-active (the only thing that proves the + cluster is storing ciphertext)." + [out] + (let [field (fn [k] (second (re-find (re-pattern (str "(?m)^\\s*" k ":\\s*(\\S+)\\s*$")) out))) + num (fn [k] (some-> (field k) Long/parseLong))] + {:full-node-id (num "full_node_id") + :local-epoch (num "local_epoch") + :storage-envelope-active (= "true" (field "storage_envelope_active"))})) + +(defn- encryption-status + [node endpoint] + (parse-encryption-status + (c/on node (c/su (c/exec admin-bin "encryption" "status" (str "--endpoint=" endpoint)))))) + +(defn- wrap-fresh-dek! + "Returns a base64 KEK-wrapped DEK, generated on the node." + [node] + (clojure.string/trim + (c/on node (c/su (c/exec encryption-setup-bin (str "--kek-file=" kek-file)))))) + +(defn- activate-encryption! + "Bootstraps the DEKs and performs the §7.1 Phase-1 storage cutover, then + VERIFIES it applied. + + The verification is the point. --encryption-enabled only enables the + EncryptionAdmin mutator RPCs; buildEncryptionWriteWiring deliberately keeps + the store's envelope gate closed until BOTH BootstrapEncryption and + EnableStorageEnvelope have applied. Without these calls every workload ran + against a cleartext cluster and could still report PASS -- which, as the §8.4 + gate's own rationale says, is worse than having no gate, because the run gets + recorded as encryption evidence. So a cluster that does not report + storage_envelope_active here must fail setup rather than proceed." + [test node grpc-port raft-groups] + (let [endpoint (encryption-endpoint node grpc-port raft-groups) + peers (map #(encryption-endpoint % grpc-port raft-groups) (:nodes test)) + wrapped-s (wrap-fresh-dek! node) + wrapped-r (wrap-fresh-dek! node)] + (info "bootstrapping encryption" endpoint) + (c/on node (c/su (apply c/exec (bootstrap-args endpoint peers wrapped-s wrapped-r)))) + (let [{:keys [full-node-id local-epoch]} (encryption-status node endpoint)] + (when-not full-node-id + (throw (ex-info "encryption status did not report a full_node_id" + {:endpoint endpoint}))) + (info "enabling storage envelope" endpoint full-node-id local-epoch) + (c/on node (c/su (apply c/exec (enable-storage-envelope-args + endpoint full-node-id (or local-epoch 0)))))) + ;; Every node must report the cutover, not just the proposer: a node that + ;; has not applied it is still writing cleartext, and the workload would be + ;; measuring a half-encrypted cluster. + (doseq [peer (:nodes test)] + (let [peer-endpoint (encryption-endpoint peer grpc-port raft-groups)] + (util/await-fn + (fn [] + (when (:storage-envelope-active (encryption-status node peer-endpoint)) + true)) + {:timeout 60000 + :log-message (str "waiting for storage envelope cutover on " peer)}))) + (info "encryption active on every node"))) + (defrecord ElastickvDB [opts] db/DB (setup! [_ test node] @@ -194,6 +361,8 @@ (c/su (c/exec :mkdir :-p data-dir) (c/exec :rm :-f log-file transport-metrics-file))) + (when (:encryption opts) + (provision-kek! node)) (start-node! test node (merge {:data-dir data-dir :grpc-port (or (:grpc-port opts) 50051) :redis-port (or (:redis-port opts) 6379) @@ -225,7 +394,14 @@ (warn t "retrying join for" peer) nil))) {:timeout 120000 - :log-message (str "joining " peer)})))) + :log-message (str "joining " peer)})) + ;; After membership, not before: the bootstrap's writer batch needs a + ;; registry entry for every member (§5.6 step 1a), and the cutover's + ;; capability gate requires every voter to report encryption-capable. + ;; Running this against a single-node cluster would register one writer + ;; and then refuse the cutover once the peers joined. + (when (:encryption opts) + (activate-encryption! test node grpc-port raft-groups)))) (info "node started" node)) (teardown! [_ _test node] diff --git a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj index 7848d051a..e94e81fdd 100644 --- a/jepsen/src/elastickv/dynamodb_multi_table_workload.clj +++ b/jepsen/src/elastickv/dynamodb_multi_table_workload.clj @@ -521,7 +521,8 @@ :dynamo-port node->port :raft-groups (:raft-groups opts) :shard-ranges (:shard-ranges opts) - :server-env (:server-env opts)})) + :server-env (:server-env opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/dynamodb_types_workload.clj b/jepsen/src/elastickv/dynamodb_types_workload.clj index fc3e6a4be..3105c69f3 100644 --- a/jepsen/src/elastickv/dynamodb_types_workload.clj +++ b/jepsen/src/elastickv/dynamodb_types_workload.clj @@ -427,7 +427,8 @@ :redis-port (or (:redis-port opts) 6379) :dynamo-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/dynamodb_workload.clj b/jepsen/src/elastickv/dynamodb_workload.clj index c5d586a43..87a0f7ad8 100644 --- a/jepsen/src/elastickv/dynamodb_workload.clj +++ b/jepsen/src/elastickv/dynamodb_workload.clj @@ -329,7 +329,8 @@ :redis-port (or (:redis-port opts) 6379) :dynamo-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/redis_workload.clj b/jepsen/src/elastickv/redis_workload.clj index 20283a64c..ac4f5d3f2 100644 --- a/jepsen/src/elastickv/redis_workload.clj +++ b/jepsen/src/elastickv/redis_workload.clj @@ -125,7 +125,8 @@ (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) :redis-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/redis_zset_safety_workload.clj b/jepsen/src/elastickv/redis_zset_safety_workload.clj index a214da5fd..0b499756b 100644 --- a/jepsen/src/elastickv/redis_zset_safety_workload.clj +++ b/jepsen/src/elastickv/redis_zset_safety_workload.clj @@ -1415,7 +1415,8 @@ (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) :redis-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 10)) time-limit (or (:time-limit opts) 60) faults (if local? diff --git a/jepsen/src/elastickv/s3_workload.clj b/jepsen/src/elastickv/s3_workload.clj index c25655cc6..317b2aa1d 100644 --- a/jepsen/src/elastickv/s3_workload.clj +++ b/jepsen/src/elastickv/s3_workload.clj @@ -187,7 +187,8 @@ :redis-port (or (:redis-port opts) 6379) :s3-port node->port :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) faults (if local? diff --git a/jepsen/src/elastickv/sqs_htfifo_workload.clj b/jepsen/src/elastickv/sqs_htfifo_workload.clj index 9942972a5..32ecaeb9d 100644 --- a/jepsen/src/elastickv/sqs_htfifo_workload.clj +++ b/jepsen/src/elastickv/sqs_htfifo_workload.clj @@ -495,7 +495,8 @@ :sqs-port node->port :sqs-region sqs-region :raft-groups (:raft-groups opts) - :shard-ranges (:shard-ranges opts)})) + :shard-ranges (:shard-ranges opts) + :encryption (:encryption opts)})) rate (double (or (:rate opts) 5)) time-limit (or (:time-limit opts) 30) ;; Drain must outlast the visibility-timeout window plus a diff --git a/jepsen/test/elastickv/encrypted_cluster_test.clj b/jepsen/test/elastickv/encrypted_cluster_test.clj new file mode 100644 index 000000000..8796b6f53 --- /dev/null +++ b/jepsen/test/elastickv/encrypted_cluster_test.clj @@ -0,0 +1,202 @@ +(ns elastickv.encrypted-cluster-test + "Pins the §8.4 encrypted acceptance gate: the EXISTING Redis and + DynamoDB workloads must be runnable against a cluster with + data-at-rest encryption enabled. + + The design is explicit that no new workload is required — encryption + is consistency-transparent, so the gate is a cluster-setup switch. The + risk that switch carries is that it silently does nothing: a run that + reported PASS while the cluster was never actually encrypted would be + worse than no gate at all, because it would be recorded as evidence." + (:require [clojure.test :refer :all] + [elastickv.cli :as cli] + [elastickv.db :as ekdb] + [elastickv.dynamodb-multi-table-workload :as multi] + [elastickv.dynamodb-types-workload :as types] + [elastickv.dynamodb-workload :as dynamo] + [elastickv.redis-workload :as redis] + [elastickv.redis-zset-safety-workload :as zset] + [elastickv.s3-workload :as s3] + [elastickv.sqs-htfifo-workload :as sqs])) + +(deftest encryption-flag-is-available-to-every-workload + ;; It lives in common-cli-opts rather than per-workload, so a future + ;; workload gets the gate without opting in. + (let [names (set (map second cli/common-cli-opts))] + (is (contains? names "--encryption")))) + +(deftest encryption-defaults-off + ;; The unencrypted suites are the existing baseline; turning this on + ;; by default would silently change what every current run measures. + (let [spec (first (filter #(= "--encryption" (second %)) cli/common-cli-opts))] + (is (false? (:default (apply hash-map (drop 3 spec))))))) + +(deftest redis-workload-propagates-encryption-to-the-db + (let [test-map (redis/elastickv-redis-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest dynamodb-workload-propagates-encryption-to-the-db + (let [test-map (dynamo/elastickv-dynamodb-test {:encryption true})] + (is (true? (get-in test-map [:db :opts :encryption]))))) + +(deftest encryption-is-absent-from-the-db-when-not-requested + ;; The flag must not leak a truthy value into an ordinary run. + (let [test-map (redis/elastickv-redis-test {})] + (is (not (true? (get-in test-map [:db :opts :encryption])))))) + +(deftest db-accepts-the-encryption-option + ;; ekdb/db carries opts verbatim; this pins that the key survives + ;; construction rather than being dropped by a destructuring form. + (let [db (ekdb/db {:grpc-port 50051 :encryption true})] + (is (true? (get-in db [:opts :encryption]))))) + +;; --------------------------------------------------------------------------- +;; The load-bearing property: the switch must actually reach the server +;; --------------------------------------------------------------------------- + +(defn- args-for [over] + (ekdb/server-args (merge {:node "n1" :grpc "n1:50051" :redis "n1:6379" + :data-dir "/var/lib/elastickv" + :raft-redis-map "n1=n1:6379"} + over))) + +(deftest encryption-emits-all-three-server-flags + ;; A sidecar path alone only enables read-only capability probing. + ;; The mutating RPCs the bootstrap needs require --encryption-enabled + ;; AND a KEK source, so all three must travel together — two of the + ;; three would produce a cluster that refuses to start, or worse, one + ;; that starts unencrypted. + (let [args (set (args-for {:encryption true}))] + (is (contains? args "--encryption-enabled")) + (is (contains? args "--encryptionSidecarPath")) + (is (contains? args "--kekFile")))) + +(deftest without-encryption-no-encryption-flag-is-emitted + ;; The unencrypted suites must be byte-identical to before, or the + ;; baseline every existing run measures has silently changed. + (let [args (set (args-for {}))] + (is (not (contains? args "--encryption-enabled"))) + (is (not (contains? args "--encryptionSidecarPath"))) + (is (not (contains? args "--kekFile"))))) + +(deftest encryption-does-not-disturb-the-other-flags + (let [plain (remove #{"--encryptionSidecarPath" "--encryption-enabled" "--kekFile"} + (args-for {:encryption true})) + encrypted (args-for {})] + ;; Removing the encryption flags and their values must leave the + ;; same argv an unencrypted run would produce. + (is (= (set encrypted) + (set (remove #(or (= % "/var/lib/elastickv/keys.json") + (= % "/var/lib/elastickv/kek.bin")) + plain)))))) + +;; --------------------------------------------------------------------------- +;; The switch must ACTIVATE encryption, not just reach the server +;; --------------------------------------------------------------------------- +;; +;; The flag tests above pass whether or not the cluster ends up encrypted: +;; --encryption-enabled only opens the EncryptionAdmin mutator RPCs, and +;; buildEncryptionWriteWiring keeps the store's envelope gate closed until BOTH +;; BootstrapEncryption and EnableStorageEnvelope have applied. Without those +;; calls every workload ran against a cleartext cluster and still reported PASS +;; -- the exact failure this namespace's docstring names as worse than no gate. + +(deftest bootstrap-discovers-the-writer-batch-from-every-member + ;; §5.6 step 1a needs one registry entry per member, with each node's real + ;; full_node_id and local_epoch. Hand-written --writer entries would go stale + ;; as soon as a node restarted and bumped its epoch, so the batch must be + ;; discovered from every node. + (let [args (ekdb/bootstrap-args "n1:50051" ["n1:50051" "n2:50051" "n3:50051"] "WS" "WR")] + (is (= 3 (count (filter #(re-find #"^--discover-from=" %) args)))) + (is (some #{"--discover-from=n2:50051"} args)) + (is (some #{"--discover-from=n3:50051"} args)))) + +(deftest bootstrap-passes-distinct-non-zero-dek-ids + ;; Bootstrap rejects a zero id and rejects the two being equal. + (let [args (ekdb/bootstrap-args "n1:50051" ["n1:50051"] "WS" "WR") + id (fn [flag] (some->> args + (filter #(clojure.string/starts-with? % (str flag "="))) + first + (re-find #"\d+$") + Long/parseLong))] + (is (pos? (id "--storage-dek-id"))) + (is (pos? (id "--raft-dek-id"))) + (is (not= (id "--storage-dek-id") (id "--raft-dek-id"))))) + +(deftest bootstrap-carries-both-wrapped-deks + (let [args (set (ekdb/bootstrap-args "n1:50051" ["n1:50051"] "WRAPPED-S" "WRAPPED-R"))] + (is (contains? args "--wrapped-storage-dek=WRAPPED-S")) + (is (contains? args "--wrapped-raft-dek=WRAPPED-R")))) + +(deftest enable-storage-envelope-carries-the-proposer-identity + ;; §6.1 treats proposer-node-id 0 as the not-capable sentinel, so the real + ;; full_node_id read back from `encryption status` has to be threaded through. + (let [args (set (ekdb/enable-storage-envelope-args "n1:50051" 12345 7))] + (is (contains? args "--proposer-node-id=12345")) + (is (contains? args "--proposer-local-epoch=7")))) + +(deftest status-parsing-reads-the-fields-the-cutover-needs + (let [out (str "capability:\n" + " encryption_capable: true\n" + " sidecar_present: true\n" + " full_node_id: 8675309\n" + " local_epoch: 3\n" + "sidecar:\n" + " storage_envelope_active: true\n") + got (ekdb/parse-encryption-status out)] + (is (= 8675309 (:full-node-id got))) + (is (= 3 (:local-epoch got))) + (is (true? (:storage-envelope-active got))))) + +(deftest status-parsing-treats-an-inactive-envelope-as-inactive + ;; The polling loop waits on this value, so a false must never read as true: + ;; that would let the workload start against a cleartext cluster, which is the + ;; whole failure mode. + (let [out (str "capability:\n" + " full_node_id: 1\n" + " local_epoch: 0\n" + "sidecar:\n" + " storage_envelope_active: false\n") + got (ekdb/parse-encryption-status out)] + (is (false? (:storage-envelope-active got))))) + +(deftest status-parsing-does-not-invent-an-active-envelope + ;; A node with no sidecar omits the line entirely. Absent must not be active. + (let [got (ekdb/parse-encryption-status "capability:\n encryption_capable: false\n")] + (is (false? (:storage-envelope-active got))) + (is (nil? (:full-node-id got))))) + +(deftest encryption-admin-targets-the-default-raft-group + ;; Bootstrap and the cutover are proposed through the default group, so a + ;; multi-group deployment must be addressed on that group's port. group-ids is + ;; sorted, so the lowest id is the default one. + (is (= "n1:50051" (ekdb/encryption-endpoint "n1" 50051 nil))) + (is (= "n1:50061" (ekdb/encryption-endpoint "n1" 50051 {1 50061, 2 50062}))) + (is (= "n1:50061" (ekdb/encryption-endpoint "n1" 50051 {2 50062, 1 50061})))) + +;; --------------------------------------------------------------------------- +;; Every workload that accepts --encryption must honour it +;; --------------------------------------------------------------------------- + +(deftest every-workload-accepting-encryption-propagates-it + ;; --encryption lives in common-cli-opts so that, as the test above puts it, + ;; "a future workload gets the gate without opting in". That promise was not + ;; kept: five entrypoints accepted the flag from the common options and then + ;; dropped it when constructing ekdb/db, so `--encryption` on those commands + ;; silently launched a cleartext cluster -- the CLI advertising a guarantee it + ;; did not provide. + ;; + ;; Driving every constructor from one list is the point: a new workload that + ;; forgets to thread the option fails here instead of shipping a silent lie. + (doseq [[label ctor] [["redis" redis/elastickv-redis-test] + ["redis-zset-safety" zset/elastickv-zset-safety-test] + ["dynamodb" dynamo/elastickv-dynamodb-test] + ["dynamodb-types" types/elastickv-dynamodb-types-test] + ["dynamodb-multi-table" multi/elastickv-dynamodb-multi-table-test] + ["s3" s3/elastickv-s3-test] + ["sqs-htfifo" sqs/elastickv-sqs-htfifo-test]]] + (testing label + (is (true? (get-in (ctor {:encryption true}) [:db :opts :encryption])) + (str label " accepts --encryption from common-cli-opts but drops it")) + (is (not (true? (get-in (ctor {}) [:db :opts :encryption]))) + (str label " must not enable encryption when it was not requested")))))