feat(orchestrate): S6 — secrets saga phase resolves + injects secret:// env (spec 04 §6) (#44)

Complete M4: the up saga now resolves every secret:// reference and injects the
values into containers without any value touching disk.

- New `secrets` phase (AlwaysRun, never cached, no global mutation): for each
  requested project it collects the env-key→ref map (generate.SecretRefs, whose
  keys match the valueless keys generate already emits), batches one Resolve per
  provider via the registry, and stashes the resolved KEY=VALUE env per project.
- compose-up injects that env via docker.Compose.Env → Compose substitutes the
  valueless keys from the child process env; the values live only in memory + the
  child env (§7.5). Registry is built from workspace.secrets.providers +
  RegisterBuiltins (SOPS+age), or injected (tests).
- generate.SecretRefs(model, project): the project's env-key→secret-ref map,
  keys identical to projectEnv's valueless keys (raw verbatim; prefixed as
  <SERVICE>_<k>).

Tests: the resolved value reaches compose-up's process env (DB_PASSWORD=…) via a
fake registry/provider, and ONLY there; the happy-path re-run treats `secrets`
as AlwaysRun. Generate determinism unchanged.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gustavobertoi

internal/generate/secrets.go

@@ -0,0 +1,34 @@
+package generate
+
+import (
+	"github.com/open-source-cloud/devstack/internal/config"
+	"github.com/open-source-cloud/devstack/internal/secrets"
+)
+
+// SecretRefs returns the compose env-key → secret:// reference for every
+// secret-valued env entry across a project's services. The KEY here is byte-for-
+// byte the valueless key projectEnv emits (env.raw verbatim; env.prefixed as
+// <SERVICE>_<k>), so the up saga can inject `KEY=<resolved>` through the
+// compose-up process env and have Compose substitute the valueless key (§7.5) —
+// no secret value ever in a generated file.
+func SecretRefs(m *config.Model, project string) map[string]string {
+	out := map[string]string{}
+	p, ok := m.Projects[project]
+	if !ok {
+		return out
+	}
+	for _, sname := range sortedKeys(p.Services) {
+		svc := p.Services[sname]
+		for k, v := range svc.Env.Raw {
+			if secrets.IsRef(v) {
+				out[k] = v
+			}
+		}
+		for k, v := range svc.Env.Prefixed {
+			if secrets.IsRef(v) {
+				out[envPrefix(sname)+"_"+k] = v
+			}
+		}
+	}
+	return out
+}

internal/orchestrate/up.go

@@ -14,6 +14,7 @@ import (
 	"github.com/open-source-cloud/devstack/internal/health"
 	"github.com/open-source-cloud/devstack/internal/hooks"
 	"github.com/open-source-cloud/devstack/internal/lock"
+	"github.com/open-source-cloud/devstack/internal/secrets"
 	"github.com/open-source-cloud/devstack/internal/state"
 	"github.com/open-source-cloud/devstack/internal/template"
 	"github.com/open-source-cloud/devstack/internal/workspace"
@@ -46,6 +47,9 @@ type UpDeps struct {
 	Env      map[string]string // generate env (nil → process env, via generate default)
 	Profile  string
 	Projects []string // explicit subset; empty → every project in the workspace
+	// Secrets resolves secret:// refs; nil → built from workspace.secrets.providers
+	// with the built-in factories (SOPS+age). Injected for tests.
+	Secrets *secrets.Registry
 
 	Build         bool          // compose up --build
 	NoHooks       bool          // skip the hooks phase
@@ -73,17 +77,23 @@ func BuildUp(d UpDeps) ([]Phase, error) {
 		return nil, err
 	}
 
+	// Resolved secret env per project, shared between the secrets phase (writes)
+	// and each compose-up phase (reads). Values live only here + in the child
+	// process env — never on disk.
+	secretEnv := map[string][]string{}
+
 	var phases []Phase
 	if !d.NoPreflight {
 		phases = append(phases, preflightPhase(d))
 	}
 	phases = append(phases,
 		networkPhase(d),
 		generatePhase(d, gen),
+		secretsPhase(d, projects, secretEnv),
 		sharedPhase(d, projects),
 	)
 	for _, p := range projects {
-		phases = append(phases, composeUpPhase(d, p))
+		phases = append(phases, composeUpPhase(d, p, secretEnv))
 	}
 	if !d.NoHooks {
 		for _, p := range projects {
@@ -93,6 +103,76 @@ func BuildUp(d UpDeps) ([]Phase, error) {
 	return phases, nil
 }
 
+// secretsPhase resolves every secret:// ref the requested projects reference and
+// stashes the resolved KEY=VALUE env per project (spec 04 §6). It ALWAYS runs
+// (never cached — values stay in memory) and mutates nothing global, so it has no
+// compensation. A nil Secrets registry with no secret refs is a no-op; refs with
+// no registry is a clear error.
+func secretsPhase(d UpDeps, projects []string, out map[string][]string) Phase {
+	return Phase{
+		Name:      "secrets",
+		AlwaysRun: true,
+		Run: func(ctx context.Context) (any, error) {
+			total := 0
+			for _, p := range projects {
+				keyRefs := generate.SecretRefs(d.Model, p)
+				if len(keyRefs) == 0 {
+					continue
+				}
+				reg, err := d.secretRegistry()
+				if err != nil {
+					return nil, err
+				}
+				raws := make([]string, 0, len(keyRefs))
+				for _, raw := range keyRefs {
+					raws = append(raws, raw)
+				}
+				refs, err := secrets.Collect(raws...)
+				if err != nil {
+					return nil, err
+				}
+				resolved, err := secrets.Resolve(ctx, reg, refs)
+				if err != nil {
+					return nil, fmt.Errorf("resolve secrets for %s: %w", p, err)
+				}
+				env := make([]string, 0, len(keyRefs))
+				for _, key := range sortedStringKeys(keyRefs) {
+					env = append(env, key+"="+resolved[keyRefs[key]])
+				}
+				out[p] = env
+				total += len(env)
+			}
+			return map[string]any{"resolved": total}, nil
+		},
+	}
+}
+
+// secretRegistry returns the injected registry, or builds one from the
+// workspace's declared providers with the built-in factories (SOPS+age).
+func (d UpDeps) secretRegistry() (*secrets.Registry, error) {
+	if d.Secrets != nil {
+		return d.Secrets, nil
+	}
+	reg := secrets.NewRegistry()
+	secrets.RegisterBuiltins(reg)
+	for _, pr := range d.Model.Workspace.Secrets.Providers {
+		reg.Configure(secrets.ProviderConfig{
+			Name: pr.Name, Kind: pr.Kind, Env: pr.Env,
+			ProjectID: pr.ProjectID, Region: pr.Region,
+		})
+	}
+	return reg, nil
+}
+
+func sortedStringKeys(m map[string]string) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
 // preflight — daemon reachable (critical). The full doctor matrix is X6.
 func preflightPhase(d UpDeps) Phase {
 	return Phase{
@@ -243,13 +323,17 @@ func gateShared(ctx context.Context, d UpDeps, names []string) ([]map[string]any
 
 // composeUpPhase brings one project stack up. Compensation tears it back down
 // (idempotent) — refs are owned by the shared phase, not unwound here.
-func composeUpPhase(d UpDeps, project string) Phase {
+func composeUpPhase(d UpDeps, project string, secretEnv map[string][]string) Phase {
 	outDir := filepath.Join(d.Model.ProjectDir(project), generate.GenDir)
 	cp := func() docker.Compose {
 		return docker.Compose{
 			Project: "devstack-" + project,
 			File:    filepath.Join(outDir, generate.ComposeFile),
 			Dir:     outDir, Runner: d.Runner,
+			// Resolved secret values reach the containers ONLY here, via the
+			// compose-up process env (Compose substitutes the valueless keys); they
+			// are never written to a file (§7.5).
+			Env: secretEnv[project],
 		}
 	}
 	return Phase{

internal/orchestrate/up_test.go

@@ -11,30 +11,45 @@ import (
 	"github.com/open-source-cloud/devstack/internal/config"
 	"github.com/open-source-cloud/devstack/internal/docker"
 	"github.com/open-source-cloud/devstack/internal/generate"
+	"github.com/open-source-cloud/devstack/internal/secrets"
 	"github.com/open-source-cloud/devstack/internal/state"
 	"github.com/open-source-cloud/devstack/internal/template"
 	"github.com/open-source-cloud/devstack/internal/workspace"
 	"github.com/open-source-cloud/devstack/templates"
 )
 
-// fakeRunner records compose invocations and can fail selectively.
+// fakeRunner records compose invocations (with their injected env) and can fail
+// selectively.
 type fakeRunner struct {
 	cmds [][]string
+	envs [][]string // env passed alongside each cmd (same index)
 	fail func(args []string) bool
 }
 
-func (f *fakeRunner) record(name string, args []string) error {
+func (f *fakeRunner) record(env []string, name string, args []string) error {
 	f.cmds = append(f.cmds, append([]string{name}, args...))
+	f.envs = append(f.envs, env)
 	if f.fail != nil && f.fail(args) {
 		return errors.New("compose failed")
 	}
 	return nil
 }
-func (f *fakeRunner) Run(_ context.Context, _ []string, _, name string, args ...string) error {
-	return f.record(name, args)
+func (f *fakeRunner) Run(_ context.Context, env []string, _, name string, args ...string) error {
+	return f.record(env, name, args)
 }
-func (f *fakeRunner) Output(_ context.Context, _ []string, _, name string, args ...string) ([]byte, error) {
-	return nil, f.record(name, args)
+func (f *fakeRunner) Output(_ context.Context, env []string, _, name string, args ...string) ([]byte, error) {
+	return nil, f.record(env, name, args)
+}
+
+// envForUp returns the env injected for a project's compose up (or nil).
+func (f *fakeRunner) envForUp(project string) []string {
+	for i, c := range f.cmds {
+		joined := strings.Join(c, " ")
+		if strings.Contains(joined, "-p "+project) && strings.Contains(joined, " up ") {
+			return f.envs[i]
+		}
+	}
+	return nil
 }
 func (f *fakeRunner) sawUp(project string) bool {
 	for _, c := range f.cmds {
@@ -163,7 +178,7 @@ func TestBuildUpHappyPath(t *testing.T) {
 	}
 	for _, r := range recs2 {
 		switch r.Phase {
-		case "preflight", "hooks":
+		case "preflight", "secrets", "hooks": // AlwaysRun phases
 			if r.Status != StatusOK {
 				t.Errorf("%s should re-run ok, got %q", r.Phase, r.Status)
 			}
@@ -225,3 +240,77 @@ func scopeSuffix(scope string) string {
 	}
 	return "@" + scope
 }
+
+// fakeSecretProvider resolves any ref to a fixed value (S6 injection test).
+type fakeSecretProvider struct{ val string }
+
+func (fakeSecretProvider) Name() string { return "fake" }
+func (p fakeSecretProvider) Resolve(_ context.Context, refs []secrets.Ref) (map[string]string, error) {
+	out := map[string]string{}
+	for _, r := range refs {
+		out[r.Raw] = p.val
+	}
+	return out, nil
+}
+
+func TestBuildUpInjectsSecretEnv(t *testing.T) {
+	root := t.TempDir()
+	write := func(rel, body string) {
+		p := filepath.Join(root, rel)
+		if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	write("workspace.yaml", "apiVersion: devstack/v1\nkind: Workspace\nname: demo\nprojects:\n  - { name: app, path: app }\n")
+	write("app/devstack.yaml", "apiVersion: devstack/v1\nkind: Project\nname: app\nservices:\n  web:\n    template: node.vite\n    env:\n      raw: { DB_PASSWORD: \"secret://vault/f#k\" }\n")
+
+	m, err := config.LoadAt(root)
+	if err != nil {
+		t.Fatalf("load: %v", err)
+	}
+	db, err := state.Open(context.Background(), filepath.Join(root, "state"), "ctx")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { db.Close() })
+	mc := &docker.MockClient{}
+	src := template.NewFSSource(templates.FS)
+	lockPath := filepath.Join(root, "lock")
+	fr := &fakeRunner{}
+
+	// Inject a registry whose provider resolves the ref to a known value.
+	reg := secrets.NewRegistry()
+	reg.RegisterFactory("fake", func(secrets.ProviderConfig) (secrets.Provider, error) {
+		return fakeSecretProvider{val: "resolved-pw"}, nil
+	})
+	reg.Configure(secrets.ProviderConfig{Name: "vault", Kind: "fake"})
+
+	d := UpDeps{
+		Model: m, DB: db, Docker: mc,
+		Manager: &workspace.Manager{Model: m, DB: db, Docker: mc, Source: src, LockPath: lockPath},
+		Source:  src, LockPath: lockPath, Runner: fr, Env: map[string]string{},
+		Secrets: reg,
+	}
+	phases, err := BuildUp(d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	saga := &Saga{Workspace: m.Workspace.Name, DB: db, LockPath: lockPath}
+	if _, err := saga.Run(context.Background(), phases); err != nil {
+		t.Fatalf("saga: %v", err)
+	}
+	// The resolved secret reaches compose up via its process env — and only there.
+	env := fr.envForUp("devstack-app")
+	found := false
+	for _, kv := range env {
+		if kv == "DB_PASSWORD=resolved-pw" {
+			found = true
+		}
+	}
+	if !found {
+		t.Errorf("compose-up env should carry DB_PASSWORD=resolved-pw, got %v", env)
+	}
+}