diff --git a/server/internal/api/apiv1/convert.go b/server/internal/api/apiv1/convert.go index 5127fb97..55fbc344 100644 --- a/server/internal/api/apiv1/convert.go +++ b/server/internal/api/apiv1/convert.go @@ -14,6 +14,7 @@ import ( api "github.com/pgEdge/control-plane/api/apiv1/gen/control_plane" "github.com/pgEdge/control-plane/server/internal/config" "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/ds" "github.com/pgEdge/control-plane/server/internal/host" "github.com/pgEdge/control-plane/server/internal/pgbackrest" "github.com/pgEdge/control-plane/server/internal/task" @@ -21,28 +22,12 @@ import ( ) // isSensitiveConfigKey returns true if the given config key name likely -// contains a secret value that should not be returned in API responses. +// contains a secret value that should not be returned in API responses. The +// canonical definition lives in the database package, shared with +// ServiceSpec.DefaultOptionalFieldsFrom, which restores these same keys from +// stored state when an update omits them. func isSensitiveConfigKey(key string) bool { - k := strings.ToLower(key) - // Use suffix matching for "token" to avoid stripping non-secret keys like - // "token_budget". Keys named exactly "token" or ending with "_token" (e.g. - // "init_token", "auth_token") are still treated as sensitive. - if k == "token" || strings.HasSuffix(k, "_token") { - return true - } - patterns := []string{ - "password", "secret", - "api_key", "apikey", "api-key", - "credential", "private_key", "private-key", - "access_key", "access-key", - "init_users", // mcp 'init_users' contains embedded passwords and must be stripped - } - for _, p := range patterns { - if strings.Contains(k, p) { - return true - } - } - return false + return database.IsSensitiveConfigKey(key) } // normalizeConfig ensures a nil config map is converted to an empty map so @@ -748,10 +733,17 @@ func apiToScripts(scripts *api.DatabaseScripts) *database.ScriptStatements { } } +// apiToDatabaseSpec validates and converts apiSpec into a database.Spec. +// existingServiceIDs should hold the service_id of every service already +// present in the stored spec (nil for a create, where nothing exists yet); it +// lets validation know which services may omit secrets that +// Spec.DefaultOptionalFieldsFrom will restore from stored state afterward, +// versus a newly added service, which must still supply its own secrets. func apiToDatabaseSpec( orchestrator config.Orchestrator, id, tID *api.Identifier, apiSpec *api.DatabaseSpec, + existingServiceIDs ds.Set[string], ) (*database.Spec, error) { var databaseID string var err error @@ -771,7 +763,7 @@ func apiToDatabaseSpec( } tenantID = &t } - if err := validateDatabaseSpec(orchestrator, databaseID, apiSpec); err != nil { + if err := validateDatabaseSpec(orchestrator, databaseID, apiSpec, existingServiceIDs); err != nil { return nil, err } diff --git a/server/internal/api/apiv1/post_init_handlers.go b/server/internal/api/apiv1/post_init_handlers.go index 59d01858..54b844d1 100644 --- a/server/internal/api/apiv1/post_init_handlers.go +++ b/server/internal/api/apiv1/post_init_handlers.go @@ -310,7 +310,9 @@ func (s *PostInitHandlers) ListDatabases(ctx context.Context, req *api.ListDatab } func (s *PostInitHandlers) CreateDatabase(ctx context.Context, req *api.CreateDatabaseRequest) (*api.CreateDatabaseResponse, error) { - spec, err := apiToDatabaseSpec(s.cfg.Orchestrator, req.ID, req.TenantID, req.Spec) + // No existing services on a create, so every service must supply its own + // secrets. + spec, err := apiToDatabaseSpec(s.cfg.Orchestrator, req.ID, req.TenantID, req.Spec, nil) if err != nil { return nil, makeInvalidInputErr(err) } @@ -378,15 +380,30 @@ func (s *PostInitHandlers) GetDatabase(ctx context.Context, req *api.GetDatabase } func (s *PostInitHandlers) UpdateDatabase(ctx context.Context, req *api.UpdateDatabasePayload) (*api.UpdateDatabaseResponse, error) { - spec, err := apiToDatabaseSpec(s.cfg.Orchestrator, &req.DatabaseID, req.Request.TenantID, req.Request.Spec) + databaseID, err := dbIdentToString(req.DatabaseID) if err != nil { - return nil, makeInvalidInputErr(err) + return nil, err } - existing, err := s.dbSvc.GetDatabase(ctx, spec.DatabaseID) + existing, err := s.dbSvc.GetDatabase(ctx, databaseID) if err != nil { return nil, apiErr(err) } + + // A service already present in the stored spec may omit secrets that GET + // stripped from a prior read (see ServiceSpec.DefaultOptionalFieldsFrom) — + // tell validation which services those are so it doesn't reject the + // omission before the merge below has a chance to restore the stored + // value. A newly added service must still supply its own secrets. + existingServiceIDs := make(ds.Set[string], len(existing.Spec.Services)) + for _, svc := range existing.Spec.Services { + existingServiceIDs.Add(svc.ServiceID) + } + + spec, err := apiToDatabaseSpec(s.cfg.Orchestrator, &req.DatabaseID, req.Request.TenantID, req.Request.Spec, existingServiceIDs) + if err != nil { + return nil, makeInvalidInputErr(err) + } // API-level update validation: // ensure that for any newly added nodes, source_node (if set) refers // only to nodes that exist in the old spec. @@ -394,7 +411,8 @@ func (s *PostInitHandlers) UpdateDatabase(ctx context.Context, req *api.UpdateDa return nil, makeInvalidInputErr(err) } // Copy optional fields from the previous spec to the current spec if they - // are unset. + // are unset. This also restores service secrets omitted from an existing + // service's config (see ServiceSpec.DefaultOptionalFieldsFrom). spec.DefaultOptionalFieldsFrom(existing.Spec) err = s.dbSvc.PopulateSpecDefaults(ctx, spec) diff --git a/server/internal/api/apiv1/validate.go b/server/internal/api/apiv1/validate.go index 78a040ba..29ebe42d 100644 --- a/server/internal/api/apiv1/validate.go +++ b/server/internal/api/apiv1/validate.go @@ -95,7 +95,11 @@ func validatePgIdentConf(lines []string, path validation.Path) []error { return errs } -func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, spec *api.DatabaseSpec) error { +// validateDatabaseSpec validates spec. existingServiceIDs holds the +// service_id of every service already present in the stored spec (nil for a +// create); a service whose ID is in this set may omit secrets that +// Spec.DefaultOptionalFieldsFrom will restore from stored state afterward. +func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, spec *api.DatabaseSpec, existingServiceIDs ds.Set[string]) error { var errs []error errs = append(errs, validateCPUs(spec.Cpus, validation.NewPath("cpus"))...) @@ -184,6 +188,11 @@ func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, s for i, svc := range spec.Services { svcPath := servicesPath.AppendArrayIndex(i) + if svc == nil { + errs = append(errs, validation.NewError(errors.New("service must not be null"), svcPath)) + continue + } + // Check for duplicate service IDs if seenServiceIDs.Has(string(svc.ServiceID)) { err := errors.New("service IDs must be unique within a database") @@ -191,7 +200,8 @@ func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, s } seenServiceIDs.Add(string(svc.ServiceID)) - errs = append(errs, validateServiceSpec(svc, svcPath, false, databaseID, spec.DatabaseUsers, seenNodeNames)...) + isExistingService := existingServiceIDs.Has(string(svc.ServiceID)) + errs = append(errs, validateServiceSpec(svc, svcPath, isExistingService, databaseID, spec.DatabaseUsers, seenNodeNames)...) } } @@ -252,6 +262,12 @@ func validateDatabaseUpdate(old *database.Spec, new *api.DatabaseSpec) error { // have no bootstrap fields (e.g. postgrest) the flag has no effect. for i, svc := range new.Services { svcPath := validation.NewPath("services", validation.ArrayIndexElement(i)) + + if svc == nil { + errs = append(errs, validation.NewError(errors.New("service must not be null"), svcPath)) + continue + } + isExistingService := existingServiceIDs.Has(string(svc.ServiceID)) errs = append(errs, validateServiceSpec(svc, svcPath, isExistingService, old.DatabaseID, new.DatabaseUsers, newNodeNames)...) @@ -577,6 +593,9 @@ func validateUniquePorts(spec *api.DatabaseSpec) []error { servicesPath := validation.NewPath("services") for i, service := range spec.Services { + if service == nil { + continue + } servicePath := servicesPath.AppendArrayIndex(i) for _, h := range service.HostIds { diff --git a/server/internal/api/apiv1/validate_test.go b/server/internal/api/apiv1/validate_test.go index 5f86959b..c52e2734 100644 --- a/server/internal/api/apiv1/validate_test.go +++ b/server/internal/api/apiv1/validate_test.go @@ -690,9 +690,10 @@ func TestValidateNode(t *testing.T) { func TestValidateDatabaseSpec(t *testing.T) { for _, tc := range []struct { - name string - spec *api.DatabaseSpec - expected []string + name string + spec *api.DatabaseSpec + existingServiceIDs ds.Set[string] + expected []string }{ { name: "valid minimal", @@ -1424,16 +1425,33 @@ func TestValidateDatabaseSpec(t *testing.T) { `"spock" must be included in shared_preload_libraries`, }, }, + { + name: "null service entry is rejected, not dereferenced", + spec: &api.DatabaseSpec{ + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + Services: []*api.ServiceSpec{nil}, + }, + expected: []string{ + `services[0]: service must not be null`, + }, + }, } { t.Run(tc.name, func(t *testing.T) { - err := validateDatabaseSpec(config.OrchestratorSwarm, "test-db", tc.spec) - if len(tc.expected) < 1 { - assert.NoError(t, err) - } else { - for _, expected := range tc.expected { - assert.ErrorContains(t, err, expected) + assert.NotPanics(t, func() { + err := validateDatabaseSpec(config.OrchestratorSwarm, "test-db", tc.spec, tc.existingServiceIDs) + if len(tc.expected) < 1 { + assert.NoError(t, err) + } else { + for _, expected := range tc.expected { + assert.ErrorContains(t, err, expected) + } } - } + }) }) } } @@ -2035,6 +2053,49 @@ func TestValidateServiceSpec(t *testing.T) { } } +func TestValidateServiceSpec_RAGUpdateOmitsAPIKey(t *testing.T) { + // PLAT-715: GET strips api_key from a RAG service's config, so a + // read-edit-write cycle resubmits the pipeline without it. On create + // (isUpdate=false) that must still fail — there's no stored value to fall + // back to. On an existing service (isUpdate=true), the omission must be + // allowed here: Spec.DefaultOptionalFieldsFrom restores the real value + // afterward, and the final, merged config is re-validated strictly + // (isUpdate=false) at deploy time. + testDBUsers := []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + } + svc := &api.ServiceSpec{ + ServiceID: "rag", + ServiceType: "rag", + Version: "latest", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "tables": []any{ + map[string]any{"table": "docs", "text_column": "content", "vector_column": "embedding"}, + }, + "embedding_llm": map[string]any{"provider": "voyage", "model": "voyage-3"}, + "rag_llm": map[string]any{"provider": "anthropic", "model": "claude-3"}, + }, + }, + }, + } + + t.Run("create requires api_key", func(t *testing.T) { + err := errors.Join(validateServiceSpec(svc, nil, false, "test-db", testDBUsers)...) + assert.ErrorContains(t, err, "embedding_llm.api_key is required") + assert.ErrorContains(t, err, "rag_llm.api_key is required") + }) + + t.Run("update on an existing service allows the omission", func(t *testing.T) { + err := errors.Join(validateServiceSpec(svc, nil, true, "test-db", testDBUsers)...) + assert.NoError(t, err) + }) +} + func TestValidateServiceSpec_NameBudget(t *testing.T) { testDBUsers := []*api.DatabaseUserSpec{ {Username: "app", DbOwner: utils.PointerTo(true)}, @@ -2452,3 +2513,14 @@ func TestValidateDatabaseUpdate_ServiceBootstrapFields(t *testing.T) { }) } } + +func TestValidateDatabaseUpdate_NullServiceEntry(t *testing.T) { + old := &database.Spec{Services: []*database.ServiceSpec{{ServiceID: "rag"}}} + newSpec := &api.DatabaseSpec{Services: []*api.ServiceSpec{nil}} + + var err error + assert.NotPanics(t, func() { + err = validateDatabaseUpdate(old, newSpec) + }) + assert.ErrorContains(t, err, "services[0]: service must not be null") +} diff --git a/server/internal/database/mcp_service_config.go b/server/internal/database/mcp_service_config.go index 31e2a2e1..4f840f80 100644 --- a/server/internal/database/mcp_service_config.go +++ b/server/internal/database/mcp_service_config.go @@ -55,11 +55,11 @@ type MCPServiceConfig struct { DisableCountRows *bool `json:"disable_count_rows,omitempty"` // Optional - knowledgebase search - KBEnabled *bool `json:"kb_enabled,omitempty"` - KBEmbeddingProvider *string `json:"kb_embedding_provider,omitempty"` - KBEmbeddingModel *string `json:"kb_embedding_model,omitempty"` - KBEmbeddingAPIKey *string `json:"kb_embedding_api_key,omitempty"` - KBDatabaseHostPath *string `json:"kb_database_host_path,omitempty"` + KBEnabled *bool `json:"kb_enabled,omitempty"` + KBEmbeddingProvider *string `json:"kb_embedding_provider,omitempty"` + KBEmbeddingModel *string `json:"kb_embedding_model,omitempty"` + KBEmbeddingAPIKey *string `json:"kb_embedding_api_key,omitempty"` + KBDatabaseHostPath *string `json:"kb_database_host_path,omitempty"` } // mcpKnownKeys is the set of all valid config keys for MCP service configuration. @@ -97,8 +97,14 @@ var validLLMProviders = []string{"anthropic", "openai", "ollama"} var validEmbeddingProviders = []string{"voyage", "openai", "ollama"} var validKBEmbeddingProviders = []string{"voyage", "openai"} -// ParseMCPServiceConfig parses and validates a config map into a typed MCPServiceConfig. -// If isUpdate is true, bootstrap-only fields (init_token, init_users) are rejected. +// ParseMCPServiceConfig parses and validates a config map into a typed +// MCPServiceConfig. If isUpdate is true, bootstrap-only fields (init_token, +// init_users) are rejected, and api_key-style secrets (anthropic_api_key, +// openai_api_key, embedding_api_key, kb_embedding_api_key) are not required: +// the caller is expected to have already restored any stored value via +// Spec.DefaultOptionalFieldsFrom before validation runs, and a key still +// missing after that is caught at deploy time, when ParseMCPServiceConfig is +// called again with isUpdate=false against the final, merged config. func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceConfig, []error) { var errs []error @@ -128,7 +134,7 @@ func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceCon embeddingModel, emErrs := optionalString(config, "embedding_model") errs = append(errs, emErrs...) - embeddingAPIKey, eakErrs := optionalString(config, "embedding_api_key") + embeddingAPIKey, eakErrs := optionalSecretString(config, "embedding_api_key", isUpdate) errs = append(errs, eakErrs...) // LLM fields: conditionally required when llm_enabled is true, @@ -157,19 +163,21 @@ func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceCon if llmProvider != "" && slices.Contains(validLLMProviders, llmProvider) { switch llmProvider { case "anthropic": - key, keyErrs := requireStringForProvider(config, "anthropic_api_key", "anthropic") + key, keyErrs := requireStringForProvider(config, "anthropic_api_key", "anthropic", isUpdate) errs = append(errs, keyErrs...) if key != "" { anthropicKey = &key } case "openai": - key, keyErrs := requireStringForProvider(config, "openai_api_key", "openai") + key, keyErrs := requireStringForProvider(config, "openai_api_key", "openai", isUpdate) errs = append(errs, keyErrs...) if key != "" { openaiKey = &key } case "ollama": - url, urlErrs := requireStringForProvider(config, "ollama_url", "ollama") + // ollama_url is not a secret (GET never strips it), so it's + // always required here regardless of isUpdate. + url, urlErrs := requireStringForProvider(config, "ollama_url", "ollama", false) errs = append(errs, urlErrs...) if url != "" { ollamaURL = &url @@ -230,7 +238,7 @@ func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceCon kbEmbeddingModel, kbemErrs := optionalString(config, "kb_embedding_model") errs = append(errs, kbemErrs...) - kbEmbeddingAPIKey, kbeakErrs := optionalString(config, "kb_embedding_api_key") + kbEmbeddingAPIKey, kbeakErrs := optionalSecretString(config, "kb_embedding_api_key", isUpdate) errs = append(errs, kbeakErrs...) kbDatabaseHostPath, kbdhpErrs := optionalString(config, "kb_database_host_path") @@ -281,8 +289,10 @@ func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceCon } else if !slices.Contains(validKBEmbeddingProviders, *kbEmbeddingProvider) { errs = append(errs, fmt.Errorf("kb_embedding_provider must be one of: %s", strings.Join(validKBEmbeddingProviders, ", "))) } else { - // voyage and openai require an API key - if kbEmbeddingAPIKey == nil { + // voyage and openai require an API key, except on an update, + // where an omitted key is expected to already have been + // restored from the stored spec before validation runs. + if !isUpdate && kbEmbeddingAPIKey == nil { errs = append(errs, fmt.Errorf("kb_embedding_api_key is required when kb_embedding_provider is %q", *kbEmbeddingProvider)) } } @@ -333,10 +343,11 @@ func ParseMCPServiceConfig(config map[string]any, isUpdate bool) (*MCPServiceCon if embeddingModel == nil { errs = append(errs, fmt.Errorf("embedding_model is required when embedding_provider is set")) } - // Provider-specific credential requirements + // Provider-specific credential requirements. api_key is not + // required on an update — see requireStringForProvider. switch *embeddingProvider { case "voyage", "openai": - if embeddingAPIKey == nil { + if !isUpdate && embeddingAPIKey == nil { errs = append(errs, fmt.Errorf("embedding_api_key is required when embedding_provider is %q", *embeddingProvider)) } case "ollama": @@ -420,10 +431,19 @@ func requireString(config map[string]any, key string) (string, []error) { return s, nil } -// requireStringForProvider extracts a required non-empty string for a specific provider. -func requireStringForProvider(config map[string]any, key, provider string) (string, []error) { +// requireStringForProvider extracts a required non-empty string for a +// specific provider. When isUpdate is true, a missing or empty value is +// allowed: the caller is expected to have already restored any stored value +// via Spec.DefaultOptionalFieldsFrom before validation runs, and a value +// still missing after that is caught at deploy time, when +// ParseMCPServiceConfig is called again with isUpdate=false against the +// final, merged config. +func requireStringForProvider(config map[string]any, key, provider string, isUpdate bool) (string, []error) { val, ok := config[key] - if !ok { + if !ok || val == nil { + if isUpdate { + return "", nil + } return "", []error{fmt.Errorf("%s is required when llm_provider is %q", key, provider)} } s, ok := val.(string) @@ -431,6 +451,9 @@ func requireStringForProvider(config map[string]any, key, provider string) (stri return "", []error{fmt.Errorf("%s must be a string", key)} } if s == "" { + if isUpdate { + return "", nil + } return "", []error{fmt.Errorf("%s must not be empty", key)} } return s, nil @@ -452,6 +475,34 @@ func optionalString(config map[string]any, key string) (*string, []error) { return &s, nil } +// optionalSecretString extracts an optional secret string from the config +// map, e.g. embedding_api_key. An explicit JSON null is always treated the +// same as the key being absent, since both mean "no value provided" for an +// optional field. An empty string is normally rejected like optionalString, +// but is also treated as absent when isUpdate is true: the caller is expected +// to have already restored any stored value via Spec.DefaultOptionalFieldsFrom +// before validation runs, and a key still missing after that is caught at +// deploy time, when ParseMCPServiceConfig is called again with +// isUpdate=false against the final, merged config. A non-string value is +// always a type error, isUpdate or not. +func optionalSecretString(config map[string]any, key string, isUpdate bool) (*string, []error) { + val, ok := config[key] + if !ok || val == nil { + return nil, nil + } + s, ok := val.(string) + if !ok { + return nil, []error{fmt.Errorf("%s must be a string", key)} + } + if s == "" { + if isUpdate { + return nil, nil + } + return nil, []error{fmt.Errorf("%s must not be empty", key)} + } + return &s, nil +} + // optionalBool extracts an optional boolean from the config map. func optionalBool(config map[string]any, key string) (*bool, []error) { val, ok := config[key] diff --git a/server/internal/database/mcp_service_config_test.go b/server/internal/database/mcp_service_config_test.go index 7febaf05..6f63f5b4 100644 --- a/server/internal/database/mcp_service_config_test.go +++ b/server/internal/database/mcp_service_config_test.go @@ -252,6 +252,18 @@ func TestParseMCPServiceConfig(t *testing.T) { assert.Contains(t, joinedErr(errs).Error(), "anthropic_api_key must not be empty") }) + t.Run("anthropic with null anthropic_api_key", func(t *testing.T) { + config := map[string]any{ + "llm_enabled": true, + "llm_provider": "anthropic", + "llm_model": "claude-3-5-sonnet-20241022", + "anthropic_api_key": nil, + } + _, errs := database.ParseMCPServiceConfig(config, false) + require.NotEmpty(t, errs) + assert.Contains(t, joinedErr(errs).Error(), "anthropic_api_key is required when llm_provider is") + }) + t.Run("openai without openai_api_key", func(t *testing.T) { config := map[string]any{ "llm_enabled": true, @@ -755,6 +767,130 @@ func TestParseMCPServiceConfig(t *testing.T) { assert.Nil(t, cfg.InitToken) assert.Nil(t, cfg.InitUsers) }) + + // PLAT-715: GET strips api_key-style secrets, so a read-edit-write + // cycle resubmits config without them. On isUpdate=true these must not + // be required — Spec.DefaultOptionalFieldsFrom is expected to have + // already restored the stored value before validation runs. + t.Run("missing anthropic_api_key is allowed", func(t *testing.T) { + config := anthropicBase() + delete(config, "anthropic_api_key") + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.AnthropicAPIKey) + }) + + t.Run("missing openai_api_key is allowed", func(t *testing.T) { + config := openaiBase() + delete(config, "openai_api_key") + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.OpenAIAPIKey) + }) + + t.Run("missing embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "embedding_provider": "voyage", + "embedding_model": "voyage-3", + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.EmbeddingAPIKey) + }) + + t.Run("missing kb_embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "kb_enabled": true, + "kb_embedding_provider": "voyage", + "kb_embedding_model": "voyage-3", + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.KBEmbeddingAPIKey) + }) + + t.Run("missing ollama_url is still rejected (not a secret GET would strip)", func(t *testing.T) { + config := map[string]any{ + "llm_enabled": true, + "llm_provider": "ollama", + "llm_model": "llama3.2", + } + _, errs := database.ParseMCPServiceConfig(config, true) + require.NotEmpty(t, errs) + assert.Contains(t, joinedErr(errs).Error(), `ollama_url is required when llm_provider is "ollama"`) + }) + + // A client might explicitly submit null (rather than omitting the key) + // or an empty string; both must be treated the same as omission here. + t.Run("explicit null anthropic_api_key is allowed", func(t *testing.T) { + config := anthropicBase() + config["anthropic_api_key"] = nil + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.AnthropicAPIKey) + }) + + t.Run("empty string anthropic_api_key is allowed", func(t *testing.T) { + config := anthropicBase() + config["anthropic_api_key"] = "" + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.AnthropicAPIKey) + }) + + t.Run("explicit null openai_api_key is allowed", func(t *testing.T) { + config := openaiBase() + config["openai_api_key"] = nil + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.OpenAIAPIKey) + }) + + t.Run("explicit null embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "embedding_provider": "voyage", + "embedding_model": "voyage-3", + "embedding_api_key": nil, + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.EmbeddingAPIKey) + }) + + t.Run("empty string embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "embedding_provider": "voyage", + "embedding_model": "voyage-3", + "embedding_api_key": "", + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.EmbeddingAPIKey) + }) + + t.Run("explicit null kb_embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "kb_enabled": true, + "kb_embedding_provider": "voyage", + "kb_embedding_model": "voyage-3", + "kb_embedding_api_key": nil, + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.KBEmbeddingAPIKey) + }) + + t.Run("empty string kb_embedding_api_key is allowed", func(t *testing.T) { + config := map[string]any{ + "kb_enabled": true, + "kb_embedding_provider": "voyage", + "kb_embedding_model": "voyage-3", + "kb_embedding_api_key": "", + } + cfg, errs := database.ParseMCPServiceConfig(config, true) + require.Empty(t, errs) + assert.Nil(t, cfg.KBEmbeddingAPIKey) + }) }) t.Run("multiple errors", func(t *testing.T) { @@ -931,11 +1067,11 @@ func TestParseMCPServiceConfig(t *testing.T) { t.Run("kb_database_host_path override", func(t *testing.T) { config := map[string]any{ - "kb_enabled": true, - "kb_embedding_provider": "voyage", - "kb_embedding_model": "voyage-3-lite", - "kb_embedding_api_key": "voy-key", - "kb_database_host_path": "/data/custom/my-kb.db", + "kb_enabled": true, + "kb_embedding_provider": "voyage", + "kb_embedding_model": "voyage-3-lite", + "kb_embedding_api_key": "voy-key", + "kb_database_host_path": "/data/custom/my-kb.db", } cfg, errs := database.ParseMCPServiceConfig(config, false) require.Empty(t, errs) diff --git a/server/internal/database/rag_service_config.go b/server/internal/database/rag_service_config.go index a51feca7..b4a85fe8 100644 --- a/server/internal/database/rag_service_config.go +++ b/server/internal/database/rag_service_config.go @@ -77,8 +77,15 @@ var ragKnownTopLevelKeys = map[string]bool{ "defaults": true, } -// ParseRAGServiceConfig parses and validates a config map into a typed RAGServiceConfig. -func ParseRAGServiceConfig(config map[string]any, _ bool) (*RAGServiceConfig, []error) { +// ParseRAGServiceConfig parses and validates a config map into a typed +// RAGServiceConfig. When isUpdate is true, api_key is not required on +// providers that normally need one: an update is expected to have already had +// omitted secrets restored from the stored spec (see +// Spec.DefaultOptionalFieldsFrom), and a key that's still missing after that +// (e.g. a brand-new pipeline) is caught at deploy time instead, when +// ParseRAGServiceConfig is called again with isUpdate=false against the final, +// merged config. +func ParseRAGServiceConfig(config map[string]any, isUpdate bool) (*RAGServiceConfig, []error) { var errs []error // Check for unknown top-level keys @@ -115,7 +122,7 @@ func ParseRAGServiceConfig(config map[string]any, _ bool) (*RAGServiceConfig, [] } seenNames := make(map[string]bool, len(cfg.Pipelines)) for i, p := range cfg.Pipelines { - errs = append(errs, validateRAGPipeline(p, i, seenNames)...) + errs = append(errs, validateRAGPipeline(p, i, seenNames, isUpdate)...) } // Validate defaults (optional) @@ -134,7 +141,7 @@ func ParseRAGServiceConfig(config map[string]any, _ bool) (*RAGServiceConfig, [] return &cfg, nil } -func validateRAGPipeline(p RAGPipeline, i int, seenNames map[string]bool) []error { +func validateRAGPipeline(p RAGPipeline, i int, seenNames map[string]bool, isUpdate bool) []error { var errs []error prefix := fmt.Sprintf("pipelines[%d]", i) @@ -158,10 +165,10 @@ func validateRAGPipeline(p RAGPipeline, i int, seenNames map[string]bool) []erro } // embedding_llm (required) - errs = append(errs, validateRAGLLMConfig(p.EmbeddingLLM, prefix+".embedding_llm", ragEmbeddingProviders)...) + errs = append(errs, validateRAGLLMConfig(p.EmbeddingLLM, prefix+".embedding_llm", ragEmbeddingProviders, isUpdate)...) // rag_llm (required) - errs = append(errs, validateRAGLLMConfig(p.RAGLLM, prefix+".rag_llm", ragLLMProviders)...) + errs = append(errs, validateRAGLLMConfig(p.RAGLLM, prefix+".rag_llm", ragLLMProviders, isUpdate)...) // token_budget (optional, > 0) if p.TokenBudget != nil && *p.TokenBudget <= 0 { @@ -199,7 +206,7 @@ func validateRAGTable(t RAGPipelineTable, prefix string, j int) []error { return errs } -func validateRAGLLMConfig(llm RAGPipelineLLMConfig, prefix string, validProviders []string) []error { +func validateRAGLLMConfig(llm RAGPipelineLLMConfig, prefix string, validProviders []string, isUpdate bool) []error { var errs []error // provider (required) @@ -215,10 +222,12 @@ func validateRAGLLMConfig(llm RAGPipelineLLMConfig, prefix string, validProvider errs = append(errs, fmt.Errorf("%s.model is required", prefix)) } - // Provider-specific: api_key required for non-ollama providers + // Provider-specific: api_key required for non-ollama providers, except on + // an update, where an omitted key is expected to already have been + // restored from the stored spec before validation runs. switch llm.Provider { case "anthropic", "openai", "voyage": - if llm.APIKey == nil || *llm.APIKey == "" { + if !isUpdate && (llm.APIKey == nil || *llm.APIKey == "") { errs = append(errs, fmt.Errorf("%s.api_key is required when provider is %q", prefix, llm.Provider)) } } diff --git a/server/internal/database/rag_service_config_test.go b/server/internal/database/rag_service_config_test.go index b9f3f7ca..652df9c5 100644 --- a/server/internal/database/rag_service_config_test.go +++ b/server/internal/database/rag_service_config_test.go @@ -121,11 +121,47 @@ func TestParseRAGServiceConfig_OllamaEmbedding(t *testing.T) { assert.Nil(t, cfg.Pipelines[0].EmbeddingLLM.APIKey) } -func TestParseRAGServiceConfig_IsUpdateIgnored(t *testing.T) { - // RAG has no bootstrap-only fields; isUpdate=true should behave identically. - cfg, errs := database.ParseRAGServiceConfig(minimalRAGConfig(), true) - require.Empty(t, errs) +func TestParseRAGServiceConfig_IsUpdateRelaxesRequiredAPIKey(t *testing.T) { + // On create (isUpdate=false), api_key is required for non-ollama providers. + config := minimalRAGConfig() + config["pipelines"].([]any)[0].(map[string]any)["embedding_llm"] = map[string]any{ + "provider": "voyage", + "model": "voyage-3", + // missing api_key + } + + _, errs := database.ParseRAGServiceConfig(config, false) + require.NotEmpty(t, errs, "api_key should be required when isUpdate is false") + + // On update (isUpdate=true), a missing api_key is allowed — the caller is + // expected to have already restored any stored value via + // Spec.DefaultOptionalFieldsFrom before validation runs; a key still + // missing after that (e.g. a brand-new pipeline) is caught at deploy time, + // when ParseRAGServiceConfig is called again with isUpdate=false. + cfg, errs := database.ParseRAGServiceConfig(config, true) + require.Empty(t, errs, "api_key should not be required when isUpdate is true") require.NotNil(t, cfg) + assert.Nil(t, cfg.Pipelines[0].EmbeddingLLM.APIKey) +} + +func TestParseRAGServiceConfig_IsUpdateStillRequiresProviderAndModel(t *testing.T) { + // isUpdate only relaxes the api_key requirement — structural checks like + // provider/model still apply. + config := map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "my-pipeline", + "tables": []any{ + map[string]any{"table": "t", "text_column": "tc", "vector_column": "vc"}, + }, + "embedding_llm": map[string]any{"provider": "not-a-real-provider"}, + "rag_llm": map[string]any{"provider": "anthropic", "model": "claude-3", "api_key": "k"}, + }, + }, + } + _, errs := database.ParseRAGServiceConfig(config, true) + require.NotEmpty(t, errs) + assert.Contains(t, errs[0].Error(), "embedding_llm.provider") } func TestParseRAGServiceConfig_MissingPipelines(t *testing.T) { diff --git a/server/internal/database/service_spec_secrets.go b/server/internal/database/service_spec_secrets.go new file mode 100644 index 00000000..76e76de3 --- /dev/null +++ b/server/internal/database/service_spec_secrets.go @@ -0,0 +1,125 @@ +package database + +import "strings" + +// IsSensitiveConfigKey returns true if the given service config key name +// likely contains a secret value that should not be returned in API +// responses, and whose omission on an update should restore the stored value +// rather than be treated as removal. Shared by the API layer (which strips +// these keys from GET responses) and DefaultOptionalFieldsFrom (which fills +// them back in from stored state when omitted from an update). +func IsSensitiveConfigKey(key string) bool { + k := strings.ToLower(key) + // Use suffix matching for "token" to avoid stripping non-secret keys like + // "token_budget". Keys named exactly "token" or ending with "_token" (e.g. + // "init_token", "auth_token") are still treated as sensitive. + if k == "token" || strings.HasSuffix(k, "_token") { + return true + } + patterns := []string{ + "password", "secret", + "api_key", "apikey", "api-key", + "credential", "private_key", "private-key", + "access_key", "access-key", + "init_users", // mcp 'init_users' contains embedded passwords and must be stripped + } + for _, p := range patterns { + if strings.Contains(k, p) { + return true + } + } + return false +} + +// isBlankConfigValue reports whether v represents "no value supplied" for a +// sensitive config field: it is either nil or an empty string. Used to check +// a candidate restoration value (typically from stored state) isn't itself +// blank, so a blank stored value is never "restored" in its place. +func isBlankConfigValue(v any) bool { + if v == nil { + return true + } + s, ok := v.(string) + return ok && s == "" +} + +// restoreSensitiveConfig is the inverse of the API layer's config scrubbing: +// it fills sensitive keys that are missing entirely from newConfig with the +// corresponding value from oldConfig, so a read-edit-write cycle on an +// unrelated field doesn't require the caller to re-supply secrets that GET +// never showed them (the API layer deletes rather than blanks stripped keys, +// so this is the common case). A key that IS present in newConfig, including +// an explicit null or empty string, is left as submitted rather than +// restored — that's how a caller unsets a sensitive value, mirroring the +// explicit-null-to-unset convention used elsewhere (e.g. Patroni's API). +// Nested objects inside arrays (e.g. RAG pipelines) are matched to their old +// counterpart by a "name" field when present, falling back to position. +func restoreSensitiveConfig(newConfig, oldConfig map[string]any) map[string]any { + if newConfig == nil { + return nil + } + out := make(map[string]any, len(newConfig)) + for k, v := range newConfig { + out[k] = restoreSensitiveValue(v, oldConfig[k]) + } + for k, old := range oldConfig { + if _, present := newConfig[k]; present { + continue + } + if IsSensitiveConfigKey(k) && !isBlankConfigValue(old) { + out[k] = old + } + } + return out +} + +func restoreSensitiveValue(newVal, oldVal any) any { + switch nv := newVal.(type) { + case map[string]any: + ov, _ := oldVal.(map[string]any) + return restoreSensitiveConfig(nv, ov) + case []any: + ov, _ := oldVal.([]any) + oldByName := make(map[string]map[string]any, len(ov)) + for _, elem := range ov { + if em, ok := elem.(map[string]any); ok { + if name, ok := em["name"].(string); ok { + oldByName[name] = em + } + } + } + out := make([]any, len(nv)) + for i, elem := range nv { + em, ok := elem.(map[string]any) + if !ok { + out[i] = elem + continue + } + var match map[string]any + if name, ok := em["name"].(string); ok { + match = oldByName[name] + } else if i < len(ov) { + match, _ = ov[i].(map[string]any) + } + out[i] = restoreSensitiveConfig(em, match) + } + return out + default: + return newVal + } +} + +// DefaultOptionalFieldsFrom will default this service's config secrets to the +// values from the given service, for any secret entirely omitted from this +// service's config. This gives service secrets (e.g. a RAG pipeline's +// api_key) the same "omitted means keep the stored value" semantics that +// User.DefaultOptionalFieldsFrom and Repository.DefaultOptionalFieldsFrom +// already provide for database user passwords and backup/restore repository +// credentials. A secret submitted as an explicit null or empty string is not +// restored — that's how a caller unsets one. +func (s *ServiceSpec) DefaultOptionalFieldsFrom(other *ServiceSpec) { + if other == nil || s.Config == nil { + return + } + s.Config = restoreSensitiveConfig(s.Config, other.Config) +} diff --git a/server/internal/database/service_spec_secrets_test.go b/server/internal/database/service_spec_secrets_test.go new file mode 100644 index 00000000..83538c76 --- /dev/null +++ b/server/internal/database/service_spec_secrets_test.go @@ -0,0 +1,249 @@ +package database_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/pgEdge/control-plane/server/internal/database" +) + +func TestIsSensitiveConfigKey(t *testing.T) { + sensitive := []string{ + "password", "ro_password", "rw_password", + "secret", "client_secret", + "token", "init_token", "auth_token", + "api_key", "openai_api_key", "anthropic_api_key", "embedding_api_key", + "apikey", "api-key", + "credential", "credentials", + "private_key", "private-key", + "access_key", "access-key", + "init_users", + } + for _, key := range sensitive { + assert.True(t, database.IsSensitiveConfigKey(key), "IsSensitiveConfigKey(%q) should be true", key) + } + + notSensitive := []string{ + "token_budget", "top_n", "llm_model", "llm_provider", + "database_name", "host", "port", "table", "vector_column", + "text_column", "description", "pipeline_name", + } + for _, key := range notSensitive { + assert.False(t, database.IsSensitiveConfigKey(key), "IsSensitiveConfigKey(%q) should be false", key) + } +} + +func TestServiceSpec_DefaultOptionalFieldsFrom(t *testing.T) { + t.Run("fills api_key omitted from an existing RAG pipeline", func(t *testing.T) { + current := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + "api_key": "voyage-secret", + }, + }, + }, + }, + } + newSvc := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + }, + }, + }, + }, + } + + newSvc.DefaultOptionalFieldsFrom(current) + + pipelines := newSvc.Config["pipelines"].([]any) + embeddingLLM := pipelines[0].(map[string]any)["embedding_llm"].(map[string]any) + assert.Equal(t, "voyage-secret", embeddingLLM["api_key"]) + }) + + t.Run("a newly submitted value is not overwritten", func(t *testing.T) { + current := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": "sk-old"}} + newSvc := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": "sk-new"}} + + newSvc.DefaultOptionalFieldsFrom(current) + + assert.Equal(t, "sk-new", newSvc.Config["api_key"]) + }) + + t.Run("a pipeline with no matching old name is left without a key", func(t *testing.T) { + current := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{"name": "docs", "embedding_llm": map[string]any{"provider": "voyage", "api_key": "voyage-secret"}}, + }, + }, + } + newSvc := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{"name": "new-pipeline", "embedding_llm": map[string]any{"provider": "voyage"}}, + }, + }, + } + + newSvc.DefaultOptionalFieldsFrom(current) + + pipelines := newSvc.Config["pipelines"].([]any) + embeddingLLM := pipelines[0].(map[string]any)["embedding_llm"].(map[string]any) + _, present := embeddingLLM["api_key"] + assert.False(t, present) + }) + + t.Run("nil other is a no-op", func(t *testing.T) { + newSvc := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": ""}} + assert.NotPanics(t, func() { + newSvc.DefaultOptionalFieldsFrom(nil) + }) + assert.Equal(t, "", newSvc.Config["api_key"]) + }) + + t.Run("an explicit null clears a top-level secret instead of restoring it", func(t *testing.T) { + current := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": "sk-old"}} + newSvc := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": nil}} + + newSvc.DefaultOptionalFieldsFrom(current) + + assert.Nil(t, newSvc.Config["api_key"]) + }) + + t.Run("an explicit empty string clears a top-level secret instead of restoring it", func(t *testing.T) { + current := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": "sk-old"}} + newSvc := &database.ServiceSpec{ServiceID: "rag", Config: map[string]any{"api_key": ""}} + + newSvc.DefaultOptionalFieldsFrom(current) + + assert.Equal(t, "", newSvc.Config["api_key"]) + }) + + t.Run("an explicit null clears a nested pipeline secret instead of restoring it", func(t *testing.T) { + current := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + "api_key": "voyage-secret", + }, + }, + }, + }, + } + newSvc := &database.ServiceSpec{ + ServiceID: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + "api_key": nil, + }, + }, + }, + }, + } + + newSvc.DefaultOptionalFieldsFrom(current) + + pipelines := newSvc.Config["pipelines"].([]any) + embeddingLLM := pipelines[0].(map[string]any)["embedding_llm"].(map[string]any) + assert.Nil(t, embeddingLLM["api_key"]) + }) +} + +func TestSpec_DefaultOptionalFieldsFrom_Services(t *testing.T) { + t.Run("fills api_key omitted from a RAG service that already exists", func(t *testing.T) { + current := &database.Spec{ + Services: []*database.ServiceSpec{ + { + ServiceID: "rag1", + ServiceType: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + "api_key": "voyage-secret", + }, + }, + }, + }, + }, + }, + } + newSpec := &database.Spec{ + Services: []*database.ServiceSpec{ + { + ServiceID: "rag1", + ServiceType: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{ + "name": "docs", + "embedding_llm": map[string]any{ + "provider": "voyage", + "model": "voyage-3", + }, + }, + }, + }, + }, + }, + } + + newSpec.DefaultOptionalFieldsFrom(current) + + pipelines := newSpec.Services[0].Config["pipelines"].([]any) + embeddingLLM := pipelines[0].(map[string]any)["embedding_llm"].(map[string]any) + assert.Equal(t, "voyage-secret", embeddingLLM["api_key"]) + }) + + t.Run("a newly added service (no match in current) is left untouched", func(t *testing.T) { + current := &database.Spec{Services: []*database.ServiceSpec{}} + newSpec := &database.Spec{ + Services: []*database.ServiceSpec{ + { + ServiceID: "rag2", + ServiceType: "rag", + Config: map[string]any{ + "pipelines": []any{ + map[string]any{"name": "docs", "embedding_llm": map[string]any{"provider": "voyage"}}, + }, + }, + }, + }, + } + + newSpec.DefaultOptionalFieldsFrom(current) + + pipelines := newSpec.Services[0].Config["pipelines"].([]any) + embeddingLLM := pipelines[0].(map[string]any)["embedding_llm"].(map[string]any) + _, present := embeddingLLM["api_key"] + assert.False(t, present) + }) +} diff --git a/server/internal/database/spec.go b/server/internal/database/spec.go index 8d6fbf0c..32a3bd2e 100644 --- a/server/internal/database/spec.go +++ b/server/internal/database/spec.go @@ -451,6 +451,7 @@ func (s *Spec) DefaultOptionalFieldsFrom(other *Spec) { s.defaultOptionalFieldFromNodes(other.Nodes) s.defaultOptionalFieldFromUsers(other.DatabaseUsers) + s.defaultOptionalFieldFromServices(other.Services) if s.BackupConfig != nil && other.BackupConfig != nil { s.BackupConfig.DefaultOptionalFieldsFrom(other.BackupConfig) @@ -540,6 +541,20 @@ func (s Spec) defaultOptionalFieldFromUsers(other []*User) { } } +func (s Spec) defaultOptionalFieldFromServices(other []*ServiceSpec) { + otherServicesByID := make(map[string]*ServiceSpec, len(other)) + for _, svc := range other { + otherServicesByID[svc.ServiceID] = svc + } + + for _, svc := range s.Services { + otherSvc, ok := otherServicesByID[svc.ServiceID] + if ok { + svc.DefaultOptionalFieldsFrom(otherSvc) + } + } +} + func InstanceIDFor(hostID, databaseID, nodeName string) string { // We're using a shortened hash of the host ID to strike a compromise // between readability and global uniqueness.