From bb1254a9e26030cc6196cf94506dc637a2046e45 Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 1 Sep 2026 06:29:17 -0700 Subject: [PATCH 1/2] fix: instance health reflects in db state Updated storedToDtabase to inspect the database's instance and update the database to degraded if instance states indicate this is appropriate. Delete database modified to check "from" state via the stored state rather than etcd to prevent mismatch. PLAT-333 --- .../internal/api/apiv1/post_init_handlers.go | 6 ++- server/internal/database/database.go | 21 +++++++- server/internal/database/database_test.go | 54 +++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 server/internal/database/database_test.go diff --git a/server/internal/api/apiv1/post_init_handlers.go b/server/internal/api/apiv1/post_init_handlers.go index 59d01858..caa8491c 100644 --- a/server/internal/api/apiv1/post_init_handlers.go +++ b/server/internal/api/apiv1/post_init_handlers.go @@ -475,7 +475,11 @@ func (s *PostInitHandlers) DeleteDatabase(ctx context.Context, req *api.DeleteDa return nil, ErrDatabaseNotModifiable } - prevState := db.State + // Use the raw stored state as the guard for this transition. + prevState, err := s.dbSvc.GetStoredDatabaseState(ctx, db.DatabaseID) + if err != nil { + return nil, apiErr(err) + } err = s.dbSvc.UpdateDatabaseState(ctx, db.DatabaseID, prevState, database.DatabaseStateDeleting) if err != nil { return nil, apiErr(err) diff --git a/server/internal/database/database.go b/server/internal/database/database.go index d550fde8..51f79f89 100644 --- a/server/internal/database/database.go +++ b/server/internal/database/database.go @@ -71,13 +71,31 @@ func databaseToStored(d *Database) *StoredDatabase { } } +var degradedInstanceStates = ds.NewSet( + InstanceStateDegraded, + InstanceStateFailed, + InstanceStateUnknown, + InstanceStateStopped, +) + func storedToDatabase(d *StoredDatabase, storedSpec *StoredSpec, instances []*Instance, serviceInstances []*ServiceInstance) *Database { + state := d.State + + if state == DatabaseStateAvailable { + for _, instance := range instances { + if degradedInstanceStates.Has(instance.State) { + state = DatabaseStateDegraded + break + } + } + } + return &Database{ DatabaseID: d.DatabaseID, TenantID: d.TenantID, CreatedAt: d.CreatedAt, UpdatedAt: d.UpdatedAt, - State: d.State, + State: state, Spec: storedSpec.Spec, Instances: instances, ServiceInstances: serviceInstances, @@ -95,7 +113,6 @@ func storedToDatabases(storedDbs []*StoredDatabase, storedSpecs []*StoredSpec, a for _, instance := range allInstances { instancesByID[instance.DatabaseID] = append(instancesByID[instance.DatabaseID], instance) } - serviceInstancesByID := make(map[string][]*ServiceInstance, len(allServiceInstances)) for _, serviceInstance := range allServiceInstances { serviceInstancesByID[serviceInstance.DatabaseID] = append(serviceInstancesByID[serviceInstance.DatabaseID], serviceInstance) diff --git a/server/internal/database/database_test.go b/server/internal/database/database_test.go new file mode 100644 index 00000000..f3c6e677 --- /dev/null +++ b/server/internal/database/database_test.go @@ -0,0 +1,54 @@ +package database + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStoredToDatabaseDegradesWhenInstanceUnavailable(t *testing.T) { + tests := []struct { + name string + instanceState InstanceState + wantState DatabaseState + }{ + {"available instance keeps database available", InstanceStateAvailable, DatabaseStateAvailable}, + {"degraded instance degrades database", InstanceStateDegraded, DatabaseStateDegraded}, + {"failed instance degrades database", InstanceStateFailed, DatabaseStateDegraded}, + {"unknown instance degrades database", InstanceStateUnknown, DatabaseStateDegraded}, + {"stopped instance degrades database", InstanceStateStopped, DatabaseStateDegraded}, + {"creating instance does not degrade database", InstanceStateCreating, DatabaseStateAvailable}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stored := &StoredDatabase{ + DatabaseID: "db1", + State: DatabaseStateAvailable, + } + storedSpec := &StoredSpec{} + instances := []*Instance{ + {InstanceID: "i1", DatabaseID: "db1", State: tt.instanceState}, + } + + db := storedToDatabase(stored, storedSpec, instances, nil) + + assert.Equal(t, tt.wantState, db.State) + }) + } +} + +func TestStoredToDatabaseDoesNotDegradeNonAvailableDatabase(t *testing.T) { + stored := &StoredDatabase{ + DatabaseID: "db1", + State: DatabaseStateCreating, + } + storedSpec := &StoredSpec{} + instances := []*Instance{ + {InstanceID: "i1", DatabaseID: "db1", State: InstanceStateFailed}, + } + + db := storedToDatabase(stored, storedSpec, instances, nil) + + assert.Equal(t, DatabaseStateCreating, db.State) +} From ef92a6490c90f5cd98afb7066fee501c718fa24c Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 8 Sep 2026 09:51:12 -0700 Subject: [PATCH 2/2] fix: check Database.RawState in deletion Add a RawState field to Database, read that state in DeleteDatabase efficiently. --- server/internal/api/apiv1/post_init_handlers.go | 5 +---- server/internal/database/database.go | 6 ++++++ server/internal/database/service.go | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/internal/api/apiv1/post_init_handlers.go b/server/internal/api/apiv1/post_init_handlers.go index caa8491c..ba5a97ff 100644 --- a/server/internal/api/apiv1/post_init_handlers.go +++ b/server/internal/api/apiv1/post_init_handlers.go @@ -476,10 +476,7 @@ func (s *PostInitHandlers) DeleteDatabase(ctx context.Context, req *api.DeleteDa } // Use the raw stored state as the guard for this transition. - prevState, err := s.dbSvc.GetStoredDatabaseState(ctx, db.DatabaseID) - if err != nil { - return nil, apiErr(err) - } + prevState := db.RawState err = s.dbSvc.UpdateDatabaseState(ctx, db.DatabaseID, prevState, database.DatabaseStateDeleting) if err != nil { return nil, apiErr(err) diff --git a/server/internal/database/database.go b/server/internal/database/database.go index 51f79f89..c1c250ca 100644 --- a/server/internal/database/database.go +++ b/server/internal/database/database.go @@ -48,6 +48,11 @@ type Database struct { CreatedAt time.Time UpdatedAt time.Time State DatabaseState + // RawState is the state as persisted in storage, before any derived + // adjustments (e.g. degrading State based on instance health). Callers + // that need to perform a guarded transition against the stored state + // (e.g. a compare-and-swap) should use this instead of State. + RawState DatabaseState Spec *Spec Instances []*Instance ServiceInstances []*ServiceInstance @@ -96,6 +101,7 @@ func storedToDatabase(d *StoredDatabase, storedSpec *StoredSpec, instances []*In CreatedAt: d.CreatedAt, UpdatedAt: d.UpdatedAt, State: state, + RawState: d.State, Spec: storedSpec.Spec, Instances: instances, ServiceInstances: serviceInstances, diff --git a/server/internal/database/service.go b/server/internal/database/service.go index 29d10951..0335b542 100644 --- a/server/internal/database/service.go +++ b/server/internal/database/service.go @@ -77,6 +77,7 @@ func (s *Service) CreateDatabase(ctx context.Context, spec *Spec) (*Database, er CreatedAt: now, UpdatedAt: now, State: DatabaseStateCreating, + RawState: DatabaseStateCreating, Spec: spec, NotCreated: true, }