snapshotoffload: wire the scheduler into the server runtime

cmd/elastickv-snapshot-offload/main.go

@@ -10,6 +10,7 @@ import (
 	"log/slog"
 	"os"
 	"sort"
+	"strconv"
 	"strings"
 
 	"github.com/bootjp/elastickv/internal/raftengine/etcd"
@@ -58,6 +59,11 @@ type restoreConfig struct {
 	manifestKey string
 	dataDir     string
 	peerCSV     string
+	// expectGroupRaw is the flag text; empty means the operator did not
+	// supply one, which is an error. Parsed into expectGroupID because
+	// group 0 is a real group and cannot double as "unset".
+	expectGroupRaw string
+	expectGroupID  uint64
 }
 
 func main() {
@@ -151,6 +157,10 @@ func parseRestoreFlags(argv []string) (*restoreConfig, error) {
 	fs.StringVar(&cfg.manifestKey, "manifest-key", "", "Object key of the snapshot manifest to restore (required)")
 	fs.StringVar(&cfg.dataDir, "data-dir", "", "Fresh target raft data directory to create (required; must not already exist)")
 	fs.StringVar(&cfg.peerCSV, "peers", "", "Comma-separated raft peers id=addr,id=addr (required)")
+	fs.StringVar(&cfg.expectGroupRaw, "expect-group", "",
+		"Raft group id this data dir is for (required). The restore is refused if the manifest "+
+			"belongs to a different group, which is otherwise undetectable: nothing downstream "+
+			"records the group, so the wrong group's FSM would start under this group's identity.")
 	if err := fs.Parse(argv); err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -166,6 +176,14 @@ func parseRestoreFlags(argv []string) (*restoreConfig, error) {
 	if strings.TrimSpace(cfg.peerCSV) == "" {
 		return nil, errors.New("--peers is required")
 	}
+	if strings.TrimSpace(cfg.expectGroupRaw) == "" {
+		return nil, errors.New("--expect-group is required")
+	}
+	groupID, err := strconv.ParseUint(strings.TrimSpace(cfg.expectGroupRaw), 10, 64)
+	if err != nil {
+		return nil, errors.Wrapf(err, "parse --expect-group %q", cfg.expectGroupRaw)
+	}
+	cfg.expectGroupID = groupID
 	if err := validateStoreFlags(cfg.store); err != nil {
 		return nil, err
 	}
@@ -259,10 +277,11 @@ func runRestore(ctx context.Context, cfg *restoreConfig, logger *slog.Logger) er
 		return err
 	}
 	result, err := snapshotoffload.RestorePhysicalSnapshot(ctx, snapshotoffload.RestoreOptions{
-		Store:       store,
-		ManifestKey: cfg.manifestKey,
-		DataDir:     cfg.dataDir,
-		Peers:       peers,
+		Store:         store,
+		ManifestKey:   cfg.manifestKey,
+		DataDir:       cfg.dataDir,
+		Peers:         peers,
+		ExpectGroupID: &cfg.expectGroupID,
 	})
 	if err != nil {
 		return errors.Wrap(err, "restore physical snapshot")

cmd/elastickv-snapshot-offload/main_test.go

@@ -50,6 +50,9 @@ func TestSnapshotOffloadCLIPublishAndRestoreLocal(t *testing.T) {
 		"--manifest-key", manifest.ManifestKey,
 		"--data-dir", restoreDataDir,
 		"--peers", "n2=127.0.0.1:12002",
+		// The publish above used --group-id 2, so this is the group this
+		// data dir is for.
+		"--expect-group", "2",
 	}, io.Discard, logger)
 	require.NoError(t, err)
 	require.Equal(t, exitSuccess, code)
@@ -168,6 +171,14 @@ func TestClassifyErrorKeepsMissingSnapshotAsADataError(t *testing.T) {
 			err:  errors.Wrap(snapshotoffload.ErrInvalidOptions, "publish"),
 			want: exitUserErr,
 		},
+		{
+			// The snapshot is intact; the operator named the wrong
+			// manifest for this data dir. Automation keys off the
+			// difference, so this must not be reported as bad data.
+			name: "wrong group's manifest",
+			err:  errors.Wrap(snapshotoffload.ErrRestoreGroupMismatch, "restore"),
+			want: exitUserErr,
+		},
 	}
 
 	for _, tc := range tests {
@@ -177,3 +188,112 @@ func TestClassifyErrorKeepsMissingSnapshotAsADataError(t *testing.T) {
 		})
 	}
 }
+
+// TestSnapshotOffloadCLIRestoreRefusesAnotherGroupsManifest is the regression
+// test for a silent mis-restore.
+//
+// Nothing downstream of the restore records which group the data belongs to:
+// the prepared artifacts carry index, term, peers and payload hash, and startup
+// derives the group from the directory layout. So an operator repeating this
+// command across groups and pasting the wrong manifest key produced a
+// valid-looking directory that startup then loaded under a different group's
+// routing identity, with no error at any point. --expect-group is the only
+// place that mistake is detectable.
+func TestSnapshotOffloadCLIRestoreRefusesAnotherGroupsManifest(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	root := t.TempDir()
+	objectRoot := filepath.Join(root, "objects")
+	sourceDataDir := seedCLISnapshot(t, root, []byte("EKVTHLC1group-2-payload"), 60, 9)
+
+	var stdout bytes.Buffer
+	code, err := run(ctx, []string{
+		commandPublish,
+		"--store", storeLocal,
+		"--local-root", objectRoot,
+		"--data-dir", sourceDataDir,
+		"--prefix", "cluster-cli",
+		"--group-id", "2",
+		"--source-cluster", "cluster-cli",
+		"--binary-version", "test-version",
+	}, &stdout, logger)
+	require.NoError(t, err)
+	require.Equal(t, exitSuccess, code)
+
+	manifest, err := snapshotoffload.DecodeManifest(stdout.Bytes())
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), manifest.GroupID)
+
+	restoreDataDir := filepath.Join(root, "restored-as-group-1")
+	code, err = run(ctx, []string{
+		commandRestore,
+		"--store", storeLocal,
+		"--local-root", objectRoot,
+		"--manifest-key", manifest.ManifestKey,
+		"--data-dir", restoreDataDir,
+		"--peers", "n1=127.0.0.1:12001",
+		// The operator means group 1, but pasted group 2's manifest key.
+		"--expect-group", "1",
+	}, io.Discard, logger)
+	require.Error(t, err)
+	require.ErrorIs(t, err, snapshotoffload.ErrRestoreGroupMismatch)
+	// exitUserErr, not exitDataErr: the snapshot data is fine, the
+	// invocation named the wrong manifest. Automation distinguishes the two.
+	require.Equal(t, exitUserErr, code)
+
+	// Nothing may be left behind: the check runs before the download and
+	// before the destination is created, so a mistaken key costs nothing.
+	_, statErr := os.Stat(restoreDataDir)
+	require.True(t, os.IsNotExist(statErr),
+		"a refused restore must not create the destination")
+}
+
+// --expect-group is required, because defaulting it would silently accept
+// whatever group the manifest happens to name -- which is the behaviour the
+// flag exists to remove.
+func TestSnapshotOffloadCLIRestoreRequiresAnExpectedGroup(t *testing.T) {
+	t.Parallel()
+
+	_, err := parseRestoreFlags([]string{
+		"--store", storeLocal,
+		"--local-root", "/tmp/objects",
+		"--manifest-key", "k",
+		"--data-dir", "/tmp/restored",
+		"--peers", "n1=127.0.0.1:12001",
+	})
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "--expect-group is required")
+}
+
+// Group 0 is the dedicated TSO group, so it has to be accepted as an explicit
+// value rather than treated as "unset".
+func TestSnapshotOffloadCLIRestoreAcceptsGroupZero(t *testing.T) {
+	t.Parallel()
+
+	cfg, err := parseRestoreFlags([]string{
+		"--store", storeLocal,
+		"--local-root", "/tmp/objects",
+		"--manifest-key", "k",
+		"--data-dir", "/tmp/restored",
+		"--peers", "n1=127.0.0.1:12001",
+		"--expect-group", "0",
+	})
+	require.NoError(t, err)
+	require.Equal(t, uint64(0), cfg.expectGroupID)
+}
+
+func TestSnapshotOffloadCLIRestoreRejectsANonNumericGroup(t *testing.T) {
+	t.Parallel()
+
+	_, err := parseRestoreFlags([]string{
+		"--store", storeLocal,
+		"--local-root", "/tmp/objects",
+		"--manifest-key", "k",
+		"--data-dir", "/tmp/restored",
+		"--peers", "n1=127.0.0.1:12001",
+		"--expect-group", "one",
+	})
+	require.Error(t, err)
+}

docs/design/2026_07_19_partial_physical_snapshot_object_offload.md

@@ -48,7 +48,12 @@ The M1 object-store-neutral substrate now adds:
 - `cmd/elastickv-snapshot-offload publish` and `restore` for local and
   S3-backed operator workflows.
 
-The runtime scheduler and retention/GC remain pending.
+The runtime scheduler is implemented and wired into main.go, opt-in via
+`--snapshotOffloadBucket` (or `--snapshotOffloadLocalDir`). Retention/GC
+is implemented per §5. Restore drills and corruption tests are in place;
+multi-node acceptance and the §7 versioned-bucket decision remain
+pending; the operator runbook is at
+[`../snapshot_offload_operations.md`](../snapshot_offload_operations.md).
 
 ## 2. Safety boundary