From 888bbc87980549376b42a74fdedfca497861f12f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 10:32:38 +0200 Subject: [PATCH 1/6] direct: retry recreate's create while ALREADY_EXISTS clears A synced table's delete drops its destination Postgres table asynchronously, on a slower schedule than the synced-table record that WaitAfterDelete polls. So even after the delete-wait completes, the re-create can fail with 409 ALREADY_EXISTS ("Destination table ... already exists"), as seen on an AWS nightly (#6728 waited on the wrong object). Recreate now keeps WaitAfterDelete as the fast path and, if the create still fails with ALREADY_EXISTS, retries it until the teardown finishes (bounded at 5m) before surfacing the result. A create that fails for any other reason, or a resource that never re-creates the same id (e.g. job_runs), is returned immediately. The testserver models the race under eventual consistency: a deleted synced table disappears from GET at once but the next create for that id conflicts once. The recreate_capped test becomes recreate_conflict_retry, asserting the retry succeeds. Co-authored-by: Isaac --- .../bundles/postgres-synced-table-recreate.md | 2 +- .../recreate_capped/script | 16 ---- .../recreate_capped/test.toml | 11 --- .../databricks.yml.tmpl | 2 +- .../out.test.toml | 0 .../output.txt | 20 ++--- .../recreate_conflict_retry/script | 16 ++++ .../recreate_conflict_retry/test.toml | 9 +++ bundle/direct/apply.go | 29 ++++++- libs/testserver/fake_workspace.go | 79 ++++++++++--------- libs/testserver/postgres.go | 25 +++--- 11 files changed, 122 insertions(+), 87 deletions(-) delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_capped/script delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_capped/test.toml rename acceptance/bundle/resources/postgres_synced_tables/{recreate_capped => recreate_conflict_retry}/databricks.yml.tmpl (95%) rename acceptance/bundle/resources/postgres_synced_tables/{recreate_capped => recreate_conflict_retry}/out.test.toml (100%) rename acceptance/bundle/resources/postgres_synced_tables/{recreate_capped => recreate_conflict_retry}/output.txt (57%) create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/test.toml diff --git a/.nextchanges/bundles/postgres-synced-table-recreate.md b/.nextchanges/bundles/postgres-synced-table-recreate.md index 5de2879f1f6..1deb546ad59 100644 --- a/.nextchanges/bundles/postgres-synced-table-recreate.md +++ b/.nextchanges/bundles/postgres-synced-table-recreate.md @@ -1 +1 @@ -* Fix recreating a postgres synced table sometimes failing with a 409 ALREADY_EXISTS error while the previous table is still being deleted. ([#6728](https://github.com/databricks/cli/pull/6728)) +* Fix recreating a postgres synced table sometimes failing with a 409 ALREADY_EXISTS error while the previous table is still being deleted. ([#6728](https://github.com/databricks/cli/pull/6728), [#6748](https://github.com/databricks/cli/pull/6748)) diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/script deleted file mode 100644 index eef8b8b1fba..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/script +++ /dev/null @@ -1,16 +0,0 @@ -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -export TIMESERIES_KEY=tpep_pickup_datetime -envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle deploy - -title "The delete never completes; the capped wait times out and the recreate fails" -# The testserver leaves the synced table in DELETING forever, so the post-delete poll -# would wait indefinitely. Capping it to 1s makes the wait time out, which fails the -# recreate. errcode keeps the script going so cleanup still runs. -export TIMESERIES_KEY=tpep_dropoff_datetime -envsubst < databricks.yml.tmpl > databricks.yml -trace DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=1 errcode $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/test.toml deleted file mode 100644 index 3c269239685..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/test.toml +++ /dev/null @@ -1,11 +0,0 @@ -# Local-only, direct-only: exercises the DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT cap on the -# synced-table recreate delete-wait. INJECT_STALE_ON_DIRECT makes the testserver's synced -# table deletion never complete (it stays DELETING), so the post-delete poll would wait -# forever; capping it to 1s makes the wait time out, which fails the recreate. Verifies -# the cap and the resulting error are rendered. -# Not run on cloud (would need a real source table) or on terraform (recreates without -# polling — the flag is direct-only). The recreate itself is covered against a real -# workspace by ../recreate. -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -Env.INJECT_STALE_ON_DIRECT = "1" diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl similarity index 95% rename from acceptance/bundle/resources/postgres_synced_tables/recreate_capped/databricks.yml.tmpl rename to acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl index 15a14677bb8..4ad6fa685ed 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl @@ -1,5 +1,5 @@ bundle: - name: recreate-capped-postgres-synced-table-$UNIQUE_NAME + name: recreate-conflict-postgres-synced-table-$UNIQUE_NAME sync: paths: [] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/out.test.toml similarity index 100% rename from acceptance/bundle/resources/postgres_synced_tables/recreate_capped/out.test.toml rename to acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/out.test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt similarity index 57% rename from acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt rename to acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt index 0ba65f672e9..2b7c2ddd45e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_capped/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-capped-postgres-synced-table-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... Created postgres_catalogs.my_catalog Created postgres_projects.my_project Created postgres_synced_tables.my_table @@ -8,19 +8,19 @@ Created schemas.pipeline_storage Files: 0 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged -=== The delete never completes; the capped wait times out and the recreate fails ->>> DATABRICKS_BUNDLE_RESOURCE_MAX_WAIT=1 errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-capped-postgres-synced-table-[UNIQUE_NAME]/default/files... -Error: cannot recreate resources.postgres_synced_tables.my_table: waiting after deleting id=synced_tables/lakebase_test_[UNIQUE_NAME].public.trips_synced: timed out: synced table still exists, waiting for deletion to complete - +=== The create races the still-finishing delete (409 ALREADY_EXISTS); recreate retries and succeeds +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Warn: deploying resources.postgres_synced_tables.my_table: Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: Failing setup of Delta sync table: Destination table lakebase_test_[UNIQUE_NAME].public.trips_synced already exists +Recreated postgres_synced_tables.my_table Files: 0 uploaded, 0 deleted - -Exit code: 1 +Resources: 1 created, 0 changed, 1 deleted, 3 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.postgres_catalogs.my_catalog delete resources.postgres_projects.my_project + delete resources.postgres_synced_tables.my_table delete resources.schemas.pipeline_storage This action will result in the deletion of the following UC schemas. Any underlying data may be lost: @@ -30,6 +30,6 @@ This action will result in the deletion of the following Lakebase projects along all their branches, databases, and endpoints. All data stored in them will be permanently lost: delete resources.postgres_projects.my_project -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-capped-postgres-synced-table-[UNIQUE_NAME]/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default -Destroy: 3 deleted +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script new file mode 100644 index 00000000000..af0462e92b9 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script @@ -0,0 +1,16 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +export TIMESERIES_KEY=tpep_pickup_datetime +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +title "The create races the still-finishing delete (409 ALREADY_EXISTS); recreate retries and succeeds" +# The synced-table record is gone from GET, so the delete-wait completes, but the first +# create conflicts because the destination table is still being dropped. The recreate +# retries the create, which then succeeds. +export TIMESERIES_KEY=tpep_dropoff_datetime +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/test.toml new file mode 100644 index 00000000000..0136a29cd4b --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/test.toml @@ -0,0 +1,9 @@ +# Local-only, direct-only: exercises the recreate create-retry when a synced table's +# delete is asynchronous. INJECT_STALE_ON_DIRECT makes the testserver drop the synced-table +# record immediately (so the delete-wait completes) but reject the next create for the same +# id once with 409 ALREADY_EXISTS, modelling the destination Postgres table still being +# dropped. The recreate retries the create and succeeds. Not run on cloud (needs a real +# source table) or on terraform (direct-only flag). +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +Env.INJECT_STALE_ON_DIRECT = "1" diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 41ffd4bcb8c..7013b3089db 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -7,12 +7,14 @@ import ( "fmt" "reflect" "strings" + "time" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" ) func (d *DeploymentUnit) withResourceKey(ctx context.Context) context.Context { @@ -146,9 +148,34 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) } - return d.Create(ctx, db, newState) + // The delete-wait above only observes the resource's own read (e.g. the synced-table + // record). A delete can leave *backing* objects behind that it can't see — a synced + // table's destination Postgres table is dropped on a slower schedule — so the create + // of the same id can still fail with ALREADY_EXISTS. Retry the create until that + // teardown finishes, then surface the result. A create that fails for any other + // reason is returned immediately; server-assigned-id resources never re-create the + // same id, so they never hit this. + var createErr error + _, _ = retries.Poll[struct{}](ctx, recreateConflictRetryTimeout, func() (*struct{}, *retries.Err) { + createErr = d.Create(ctx, db, newState) + switch { + case createErr == nil: + return &struct{}{}, nil + case errors.Is(createErr, apierr.ErrAlreadyExists): + log.Warnf(ctx, "Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: %s", createErr) + return nil, retries.Continues("create still conflicts with the deleting resource") + default: + return nil, retries.Halt(createErr) + } + }) + return createErr } +// recreateConflictRetryTimeout caps how long recreate retries a create that keeps +// failing with ALREADY_EXISTS because the just-deleted resource (or its backing +// objects) is still being torn down. +const recreateConflictRetryTimeout = 5 * time.Minute + func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error { if !d.Adapter.HasDoUpdate() { return fmt.Errorf("internal error: DoUpdate not implemented for resource %s", d.ResourceKey) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 44d6ccd1889..56adaa19de5 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -45,8 +45,9 @@ const ( GuestServicePrincipalTokenPrefix = "dbapi2" // EventualConsistencyTokenPrefix identifies workspaces that simulate eventual // consistency / propagation delays: the first GET after a create returns 404 - // (not yet visible), and a deleted synced table stays in DELETING instead of - // disappearing (so the direct engine's post-delete poll has a teardown to wait out). + // (not yet visible), and after a synced table is deleted its record disappears + // from GET immediately but the next create for that id still conflicts once (the + // destination table is dropped on a slower schedule). EventualConsistencyTokenPrefix = "dbapi3" UserID = "1000012345" TestDefaultClusterId = "0123-456789-cluster0" @@ -178,12 +179,17 @@ type FakeWorkspace struct { isServicePrincipal bool // eventualConsistency simulates propagation delays (see EventualConsistencyTokenPrefix). - // For synced tables it makes deletion slow: a deleted table stays in DELETING and keeps - // being returned by GET, so the direct engine's post-delete poll (WaitAfterDelete) has a - // real teardown to wait out. Off by default, so the default path and terraform (which - // recreates without polling) keep immediate deletion. + // For synced tables a delete removes the record from GET immediately but makes the next + // create for that id conflict once, modelling the destination Postgres table being + // dropped asynchronously. Off by default so the default path and terraform recreate + // without any post-delete conflict. eventualConsistency bool + // postgresSyncedTablesDeleting counts how many further creates a just-deleted synced + // table id must reject with ALREADY_EXISTS, simulating the destination Postgres table + // being dropped asynchronously after the synced-table record is already gone from GET. + postgresSyncedTablesDeleting map[string]int + directories map[string]workspace.ObjectInfo files map[string]FileEntry repoIdByPath map[string]int64 @@ -507,36 +513,37 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { State: sql.StateRunning, }, }, - ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, - VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, - VectorSearchIndexes: map[string]fakeVectorSearchIndex{}, - Repos: map[string]workspace.RepoInfo{}, - SecretScopes: map[string]workspace.SecretScope{}, - Secrets: map[string]map[string]string{}, - Acls: map[string][]workspace.AclItem{}, - Permissions: map[string]iam.ObjectPermissions{}, - Groups: map[string]iam.Group{}, - DatabaseInstances: map[string]database.DatabaseInstance{}, - DatabaseCatalogs: map[string]database.DatabaseCatalog{}, - SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, - PostgresProjects: map[string]postgres.Project{}, - PostgresBranches: map[string]postgres.Branch{}, - PostgresCatalogs: map[string]postgres.Catalog{}, - PostgresDatabases: map[string]postgres.Database{}, - PostgresEndpoints: map[string]postgres.Endpoint{}, - PostgresRoles: map[string]postgres.Role{}, - PostgresSyncedTables: map[string]postgres.SyncedTable{}, - PostgresSnapshotSchedules: map[string]postgres.SnapshotSchedule{}, - PostgresOperations: map[string]postgres.Operation{}, - postgresImplicitBranches: map[string]bool{}, - postgresImplicitEndpoints: map[string]bool{}, - clusterVenvs: map[string]*clusterEnv{}, - DmsDeployments: map[string]*DmsDeployment{}, - DmsDeploymentNodes: map[string]string{}, - Alerts: map[string]sql.AlertV2{}, - Experiments: map[string]ml.GetExperimentResponse{}, - ModelRegistryModels: map[string]ml.Model{}, - ModelRegistryModelIDs: map[string]string{}, + ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, + VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, + VectorSearchIndexes: map[string]fakeVectorSearchIndex{}, + Repos: map[string]workspace.RepoInfo{}, + SecretScopes: map[string]workspace.SecretScope{}, + Secrets: map[string]map[string]string{}, + Acls: map[string][]workspace.AclItem{}, + Permissions: map[string]iam.ObjectPermissions{}, + Groups: map[string]iam.Group{}, + DatabaseInstances: map[string]database.DatabaseInstance{}, + DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, + PostgresProjects: map[string]postgres.Project{}, + PostgresBranches: map[string]postgres.Branch{}, + PostgresCatalogs: map[string]postgres.Catalog{}, + PostgresDatabases: map[string]postgres.Database{}, + PostgresEndpoints: map[string]postgres.Endpoint{}, + PostgresRoles: map[string]postgres.Role{}, + PostgresSyncedTables: map[string]postgres.SyncedTable{}, + postgresSyncedTablesDeleting: map[string]int{}, + PostgresSnapshotSchedules: map[string]postgres.SnapshotSchedule{}, + PostgresOperations: map[string]postgres.Operation{}, + postgresImplicitBranches: map[string]bool{}, + postgresImplicitEndpoints: map[string]bool{}, + clusterVenvs: map[string]*clusterEnv{}, + DmsDeployments: map[string]*DmsDeployment{}, + DmsDeploymentNodes: map[string]string{}, + Alerts: map[string]sql.AlertV2{}, + Experiments: map[string]ml.GetExperimentResponse{}, + ModelRegistryModels: map[string]ml.Model{}, + ModelRegistryModelIDs: map[string]string{}, Clusters: map[string]compute.ClusterDetails{ // A running dedicated single-user cluster: the shape `ssh connect --cluster` // requires (ValidateClusterAccess rejects anything else), matching the cloud diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 7d32f303ba9..e552c155db0 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1684,6 +1684,12 @@ func (s *FakeWorkspace) PostgresSyncedTableCreate(req Request, syncedTableID str if _, exists := s.PostgresSyncedTables[name]; exists { return postgresErrorResponse(409, "ALREADY_EXISTS", "synced table with such id already exists") } + // Simulate the destination Postgres table being dropped asynchronously: for a few + // creates after a delete, the record is gone from GET but the create still conflicts. + if s.postgresSyncedTablesDeleting[name] > 0 { + s.postgresSyncedTablesDeleting[name]-- + return postgresErrorResponse(409, "ALREADY_EXISTS", "Failing setup of Delta sync table: Destination table "+syncedTableID+" already exists") + } table.Name = name table.Uid = nextUUID() table.CreateTime = nowTime() @@ -1720,22 +1726,19 @@ func (s *FakeWorkspace) PostgresSyncedTableGet(name string) Response { func (s *FakeWorkspace) PostgresSyncedTableDelete(name string) Response { defer s.LockUnlock()() - table, exists := s.PostgresSyncedTables[name] - if !exists { + if _, exists := s.PostgresSyncedTables[name]; !exists { return postgresNotFoundResponse("synced table") } - if !s.eventualConsistency { - delete(s.PostgresSyncedTables, name) - return Response{Body: s.createOperationLocked(name, nil)} - } + delete(s.PostgresSyncedTables, name) - if table.Status == nil { - table.Status = &postgres.SyncedTableSyncedTableStatus{} + // Under simulated eventual consistency the synced-table record disappears from GET + // immediately (so the delete-wait completes), but the destination Postgres table is + // still being dropped — so the next create for the same id conflicts once. This is + // the race observed on AWS that the recreate create-retry handles. + if s.eventualConsistency { + s.postgresSyncedTablesDeleting[name] = 1 } - table.Status.DetailedState = postgres.SyncedTableStateSyncedTableOffline - table.Status.UnityCatalogProvisioningState = postgres.ProvisioningInfoStateDeleting - s.PostgresSyncedTables[name] = table return Response{Body: s.createOperationLocked(name, nil)} } From 45f1ab033b4c4b9d3e53d388e10733815844cd69 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 11:01:24 +0200 Subject: [PATCH 2/6] direct: bump the recreate create-retry cap to 15m; drop the template from its test No measured teardown latency to size the create-retry cap, so raise it 5m->15m to match deleteIndexTimeout (the closest async-teardown analog). Convert recreate_conflict_retry off the databricks.yml.tmpl + envsubst pattern to a committed databricks.yml edited in place with update_file.py. Co-authored-by: Isaac --- .../{databricks.yml.tmpl => databricks.yml} | 12 ++++++------ .../recreate_conflict_retry/output.txt | 10 ++++++---- .../recreate_conflict_retry/script | 11 ++++------- bundle/direct/apply.go | 6 ++++-- 4 files changed, 20 insertions(+), 19 deletions(-) rename acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/{databricks.yml.tmpl => databricks.yml} (77%) diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml similarity index 77% rename from acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl rename to acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml index 4ad6fa685ed..4023f78bc61 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: recreate-conflict-postgres-synced-table-$UNIQUE_NAME + name: test-bundle sync: paths: [] @@ -7,19 +7,19 @@ sync: resources: schemas: pipeline_storage: - name: pipeline_storage_$UNIQUE_NAME + name: pipeline_storage catalog_name: main comment: "Pipeline storage for the synced-table recreate test" postgres_projects: my_project: - project_id: test-pg-proj-$UNIQUE_NAME + project_id: test-pg-proj display_name: "Test Project for Synced Table Recreate" pg_version: 17 postgres_catalogs: my_catalog: - catalog_id: lakebase_test_$UNIQUE_NAME + catalog_id: lakebase_test branch: ${resources.postgres_projects.my_project.id}/branches/production postgres_database: appdb create_database_if_missing: true @@ -27,13 +27,13 @@ resources: postgres_synced_tables: my_table: synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced - source_table_full_name: main.source_$UNIQUE_NAME.trips_source + source_table_full_name: main.source.trips_source primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb branch: ${resources.postgres_projects.my_project.id}/branches/production create_database_objects_if_missing: true - timeseries_key: $TIMESERIES_KEY + timeseries_key: tpep_pickup_datetime new_pipeline_spec: storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt index 2b7c2ddd45e..3393a840eae 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created postgres_catalogs.my_catalog Created postgres_projects.my_project Created postgres_synced_tables.my_table @@ -9,9 +9,11 @@ Files: 0 uploaded, 0 deleted Resources: 4 created, 0 changed, 0 deleted, 0 unchanged === The create races the still-finishing delete (409 ALREADY_EXISTS); recreate retries and succeeds +>>> update_file.py databricks.yml timeseries_key: tpep_pickup_datetime timeseries_key: tpep_dropoff_datetime + >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... -Warn: deploying resources.postgres_synced_tables.my_table: Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: Failing setup of Delta sync table: Destination table lakebase_test_[UNIQUE_NAME].public.trips_synced already exists +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.postgres_synced_tables.my_table: Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: Failing setup of Delta sync table: Destination table lakebase_test.public.trips_synced already exists Recreated postgres_synced_tables.my_table Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 3 unchanged @@ -30,6 +32,6 @@ This action will result in the deletion of the following Lakebase projects along all their branches, databases, and endpoints. All data stored in them will be permanently lost: delete resources.postgres_projects.my_project -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-conflict-postgres-synced-table-[UNIQUE_NAME]/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script index af0462e92b9..4f6409a806a 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/script @@ -3,14 +3,11 @@ cleanup() { } trap cleanup EXIT -export TIMESERIES_KEY=tpep_pickup_datetime -envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy title "The create races the still-finishing delete (409 ALREADY_EXISTS); recreate retries and succeeds" -# The synced-table record is gone from GET, so the delete-wait completes, but the first -# create conflicts because the destination table is still being dropped. The recreate -# retries the create, which then succeeds. -export TIMESERIES_KEY=tpep_dropoff_datetime -envsubst < databricks.yml.tmpl > databricks.yml +# Toggle a recreate-on-change field (timeseries_key) to force a recreate. The synced-table +# record is gone from GET so the delete-wait completes, but the first create conflicts +# because the destination table is still being dropped; the recreate retries and succeeds. +trace update_file.py databricks.yml "timeseries_key: tpep_pickup_datetime" "timeseries_key: tpep_dropoff_datetime" trace $CLI bundle deploy diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 7013b3089db..3d1c0c279bf 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -173,8 +173,10 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // recreateConflictRetryTimeout caps how long recreate retries a create that keeps // failing with ALREADY_EXISTS because the just-deleted resource (or its backing -// objects) is still being torn down. -const recreateConflictRetryTimeout = 5 * time.Minute +// objects) is still being torn down. No measured teardown latency to derive this +// from, so it matches deleteIndexTimeout (15m); if it's exceeded the recreate fails +// and the next deploy re-creates (state was already dropped). +const recreateConflictRetryTimeout = 15 * time.Minute func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error { if !d.Adapter.HasDoUpdate() { From 95dfae4c6085aadf31ff4a319fce447d52e3f67a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 12:32:09 +0200 Subject: [PATCH 3/6] direct: skip recreate's ALREADY_EXISTS retry when the id changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recreate create-retry waits out a slow async teardown, but only makes sense when the create re-creates the same id it just deleted. If the recreate changed a provided-id field (e.g. renamed an app, or pointed a synced table at a new synced_table_id), the create targets a different id that a separate, pre-existing resource owns — no wait can free that name, so surface the ALREADY_EXISTS immediately instead of retrying for minutes. Adds local+cloud coverage for a synced-table rename onto an existing id, plus local fail-fast coverage for apps and vector search endpoints. Co-authored-by: Isaac --- .../recreate_rename_conflict/app_a/app.py | 1 + .../recreate_rename_conflict/app_b/app.py | 1 + .../recreate_rename_conflict/databricks.yml | 11 ++++ .../recreate_rename_conflict/out.test.toml | 3 ++ .../apps/recreate_rename_conflict/output.txt | 29 ++++++++++ .../apps/recreate_rename_conflict/script | 10 ++++ .../apps/recreate_rename_conflict/test.toml | 5 ++ .../databricks.yml.tmpl | 52 ++++++++++++++++++ .../recreate_rename_conflict/out.test.toml | 5 ++ .../recreate_rename_conflict/output.txt | 47 ++++++++++++++++ .../recreate_rename_conflict/script | 28 ++++++++++ .../recreate_rename_conflict/test.toml | 5 ++ .../recreate_rename_conflict/databricks.yml | 11 ++++ .../recreate_rename_conflict/out.test.toml | 3 ++ .../recreate_rename_conflict/output.txt | 29 ++++++++++ .../recreate_rename_conflict/script | 10 ++++ .../recreate_rename_conflict/test.toml | 6 +++ bundle/direct/apply.go | 53 ++++++++++++++++--- 18 files changed, 303 insertions(+), 6 deletions(-) create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/app_a/app.py create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/app_b/app.py create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/databricks.yml create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/out.test.toml create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/script create mode 100644 acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script create mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml create mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml create mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml create mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt create mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script create mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/app_a/app.py b/acceptance/bundle/resources/apps/recreate_rename_conflict/app_a/app.py new file mode 100644 index 00000000000..f1a18139c84 --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/app_a/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/app_b/app.py b/acceptance/bundle/resources/apps/recreate_rename_conflict/app_b/app.py new file mode 100644 index 00000000000..f1a18139c84 --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/app_b/app.py @@ -0,0 +1 @@ +print("Hello world!") diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/databricks.yml b/acceptance/bundle/resources/apps/recreate_rename_conflict/databricks.yml new file mode 100644 index 00000000000..ec593938040 --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + apps: + app_a: + name: app-alpha + source_code_path: ./app_a + app_b: + name: app-beta + source_code_path: ./app_b diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/apps/recreate_rename_conflict/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt new file mode 100644 index 00000000000..2f5adeb3b6d --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt @@ -0,0 +1,29 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created apps.app_a +Created apps.app_b +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Renaming app_a onto app_b's name fails fast (no ALREADY_EXISTS retry) +>>> update_file.py databricks.yml name: app-alpha name: app-beta + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot recreate resources.apps.app_a: An app with the same name already exists: app-beta (409 RESOURCE_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/apps?no_compute=true +HTTP Status: 409 Conflict +API error_code: RESOURCE_ALREADY_EXISTS +API message: An app with the same name already exists: app-beta + +Files: 1 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.apps.app_b + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/script b/acceptance/bundle/resources/apps/recreate_rename_conflict/script new file mode 100644 index 00000000000..4b645758dcc --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/script @@ -0,0 +1,10 @@ +trace $CLI bundle deploy + +title "Renaming app_a onto app_b's name fails fast (no ALREADY_EXISTS retry)" +# name is a provided-id field, so the plan recreates app_a: delete app-alpha, then +# create app-beta — which app_b already owns. The create must surface ALREADY_EXISTS +# immediately rather than retrying, since no wait can free a name another app owns. +trace update_file.py databricks.yml "name: app-alpha" "name: app-beta" +trace musterr $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml new file mode 100644 index 00000000000..a1405e127c0 --- /dev/null +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml @@ -0,0 +1,5 @@ +Cloud = false +RecordRequests = false + +# The same-id guard that makes a rename-to-existing fail fast is a direct-engine feature. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl new file mode 100644 index 00000000000..e49a6b4a749 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl @@ -0,0 +1,52 @@ +bundle: + name: rename-conflict-postgres-synced-table-$UNIQUE_NAME + +sync: + paths: [] + +resources: + schemas: + pipeline_storage: + name: pipeline_storage_$UNIQUE_NAME + catalog_name: main + comment: "Pipeline storage for the synced-table rename-conflict test" + + postgres_projects: + my_project: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Test Project for Synced Table Rename Conflict" + pg_version: 17 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test_$UNIQUE_NAME + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true + + postgres_synced_tables: + my_table: + synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced + source_table_full_name: main.source_$UNIQUE_NAME.trips_source + primary_key_columns: ["tpep_pickup_datetime"] + scheduling_policy: SNAPSHOT + postgres_database: appdb + branch: ${resources.postgres_projects.my_project.id}/branches/production + create_database_objects_if_missing: true + timeseries_key: tpep_pickup_datetime + new_pipeline_spec: + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} + + other_table: + synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_other + source_table_full_name: main.source_$UNIQUE_NAME.trips_source + primary_key_columns: ["tpep_pickup_datetime"] + scheduling_policy: SNAPSHOT + postgres_database: appdb + branch: ${resources.postgres_projects.my_project.id}/branches/production + create_database_objects_if_missing: true + timeseries_key: tpep_pickup_datetime + new_pipeline_spec: + storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} + storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml new file mode 100644 index 00000000000..7ad46517eca --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml @@ -0,0 +1,5 @@ +Cloud = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt new file mode 100644 index 00000000000..6ef55c4299b --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt @@ -0,0 +1,47 @@ +Creating temporary source table: main.source_[UNIQUE_NAME].trips_source +{ + "full_name": "main.source_[UNIQUE_NAME]" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Created postgres_catalogs.my_catalog +Created postgres_projects.my_project +Created postgres_synced_tables.my_table +Created postgres_synced_tables.other_table +Created schemas.pipeline_storage +Files: 0 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +=== Renaming my_table onto other_table's id fails fast (no ALREADY_EXISTS retry) +>>> update_file.py databricks.yml public.trips_synced public.trips_other + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Error: cannot recreate resources.postgres_synced_tables.my_table: synced table with such id already exists (409 ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/synced_tables?synced_table_id=lakebase_test_[UNIQUE_NAME].public.trips_other +HTTP Status: 409 Conflict +API error_code: ALREADY_EXISTS +API message: synced table with such id already exists + +Files: 0 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.postgres_catalogs.my_catalog + delete resources.postgres_projects.my_project + delete resources.postgres_synced_tables.other_table + delete resources.schemas.pipeline_storage + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.pipeline_storage + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.my_project + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default + +Destroy: 4 deleted +Cleaning up temporary source table diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script new file mode 100644 index 00000000000..cde461b74bd --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script @@ -0,0 +1,28 @@ +# Per-test source table to avoid the 20-synced-tables-per-source-table limit. +echo "Creating temporary source table: main.source_$UNIQUE_NAME.trips_source" +$CLI schemas create source_$UNIQUE_NAME main -o json | jq '{full_name}' +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.source_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", + \"wait_timeout\": \"45s\" + }" > /dev/null + +cleanup() { + trace $CLI bundle destroy --auto-approve + echo "Cleaning up temporary source table" + $CLI tables delete main.source_$UNIQUE_NAME.trips_source || true + $CLI schemas delete main.source_$UNIQUE_NAME || true + rm -f "$OUT_REQUESTS" +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml +trace $CLI bundle deploy + +title "Renaming my_table onto other_table's id fails fast (no ALREADY_EXISTS retry)" +# Point my_table at other_table's synced_table_id. synced_table_id is a provided-id +# field, so the plan recreates my_table: delete the old id, then create other_table's +# id — which a separate resource already owns. The create must surface ALREADY_EXISTS +# immediately rather than retrying for minutes, since no wait can free that name. +trace update_file.py databricks.yml "public.trips_synced" "public.trips_other" +trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml new file mode 100644 index 00000000000..b151761c7e0 --- /dev/null +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml @@ -0,0 +1,5 @@ +Cloud = true + +# The same-id guard that makes a rename-to-existing fail fast (instead of retrying +# the create for minutes) is a direct-engine feature. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml new file mode 100644 index 00000000000..b9c9d8a5e04 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-bundle + +resources: + vector_search_endpoints: + endpoint_a: + name: vs-endpoint-alpha + endpoint_type: STANDARD + endpoint_b: + name: vs-endpoint-beta + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt new file mode 100644 index 00000000000..4d581fb4d51 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt @@ -0,0 +1,29 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created vector_search_endpoints.endpoint_a +Created vector_search_endpoints.endpoint_b +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Renaming endpoint_a onto endpoint_b's name fails fast (no ALREADY_EXISTS retry) +>>> update_file.py databricks.yml name: vs-endpoint-alpha name: vs-endpoint-beta + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Error: cannot recreate resources.vector_search_endpoints.endpoint_a: Vector search endpoint with name vs-endpoint-beta already exists (409 RESOURCE_ALREADY_EXISTS) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/vector-search/endpoints +HTTP Status: 409 Conflict +API error_code: RESOURCE_ALREADY_EXISTS +API message: Vector search endpoint with name vs-endpoint-beta already exists + +Files: 1 uploaded, 0 deleted + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.endpoint_b + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script new file mode 100644 index 00000000000..5826c1965d8 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script @@ -0,0 +1,10 @@ +trace $CLI bundle deploy + +title "Renaming endpoint_a onto endpoint_b's name fails fast (no ALREADY_EXISTS retry)" +# name is a provided-id field, so the plan recreates endpoint_a: delete vs-endpoint-alpha, +# then create vs-endpoint-beta — which endpoint_b already owns. The create must surface +# ALREADY_EXISTS immediately rather than retrying, since no wait can free that name. +trace update_file.py databricks.yml "name: vs-endpoint-alpha" "name: vs-endpoint-beta" +trace musterr $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml new file mode 100644 index 00000000000..8f434449bf5 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = false + +# Local-only fast guard. The cloud path for the same-id guard is covered by +# postgres_synced_tables/recreate_rename_conflict. Direct engine only (no VS terraform provider). +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 3d1c0c279bf..05b13cf54e7 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" ) @@ -48,7 +49,7 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, switch actionType { case deployplan.Recreate: - return d.Recreate(ctx, db, oldID, newState) + return d.Recreate(ctx, db, oldID, newState, planEntry) case deployplan.Update: return d.Update(ctx, db, oldID, newState, planEntry) case deployplan.UpdateWithID: @@ -108,7 +109,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return nil } -func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any) error { +func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentState, oldID string, newState any, planEntry *deployplan.PlanEntry) error { oldState, err := d.loadPersistedState(db) if err != nil { return err @@ -152,16 +153,22 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // record). A delete can leave *backing* objects behind that it can't see — a synced // table's destination Postgres table is dropped on a slower schedule — so the create // of the same id can still fail with ALREADY_EXISTS. Retry the create until that - // teardown finishes, then surface the result. A create that fails for any other - // reason is returned immediately; server-assigned-id resources never re-create the - // same id, so they never hit this. + // teardown finishes, then surface the result. + // + // Only when the recreate re-creates the *same* id, though. If it changed a + // provided-id field (e.g. renamed an app, or pointed a synced table at a new + // synced_table_id), the create targets a *different* id that a separate, + // pre-existing resource already owns — waiting cannot free that name, so surface + // the conflict immediately instead of retrying for minutes. Any non-conflict error + // is returned immediately too. + idChanged := d.recreateChangedID(planEntry) var createErr error _, _ = retries.Poll[struct{}](ctx, recreateConflictRetryTimeout, func() (*struct{}, *retries.Err) { createErr = d.Create(ctx, db, newState) switch { case createErr == nil: return &struct{}{}, nil - case errors.Is(createErr, apierr.ErrAlreadyExists): + case !idChanged && errors.Is(createErr, apierr.ErrAlreadyExists): log.Warnf(ctx, "Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: %s", createErr) return nil, retries.Continues("create still conflicts with the deleting resource") default: @@ -171,6 +178,40 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return createErr } +// recreateChangedID reports whether this recreate edits a field that composes the +// resource's id (a provided_id_field, e.g. an app's name or a synced table's +// synced_table_id). When it does, the create targets a different id than the one +// just deleted, so an ALREADY_EXISTS can only mean a separate, pre-existing resource +// owns that id — waiting never frees it. When it does not, the recreate re-creates +// the same id, so ALREADY_EXISTS can only be the just-deleted resource still tearing +// down. +func (d *DeploymentUnit) recreateChangedID(planEntry *deployplan.PlanEntry) bool { + if planEntry == nil { + return false + } + for field, ch := range planEntry.Changes { + if ch.Action == deployplan.Skip { + continue + } + path, err := structpath.ParsePath(field) + if err != nil { + continue + } + if changesProvidedID(d.Adapter.ResourceConfig(), path) || changesProvidedID(d.Adapter.GeneratedResourceConfig(), path) { + return true + } + } + return false +} + +func changesProvidedID(cfg *dresources.ResourceLifecycleConfig, path *structpath.PathNode) bool { + if cfg == nil { + return false + } + _, ok := findMatchingRule(path, cfg.ProvidedIDFields) + return ok +} + // recreateConflictRetryTimeout caps how long recreate retries a create that keeps // failing with ALREADY_EXISTS because the just-deleted resource (or its backing // objects) is still being torn down. No measured teardown latency to derive this From 4d5507ef62fbfc060e3512949bf863f2b7f70ef3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 13:08:01 +0200 Subject: [PATCH 4/6] direct: make the synced-table rename-conflict test local-only; drop redundant VS test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On cloud a synced-table create with an already-taken id replaces the existing table (replace_existing) instead of returning ALREADY_EXISTS, so the rename never conflicts there and the cloud run "succeeded" unexpectedly. The retry the same-id guard gates only matches error_code ALREADY_EXISTS (postgres synced tables); apps and vector search endpoints return RESOURCE_ALREADY_EXISTS, a sibling sentinel the retry never caught, so they already fail fast. The guard is therefore only observable against the testserver — keep the synced-table test as its local regression, and drop the vector-search test, which duplicated the existing recreate/create-fails coverage. Co-authored-by: Isaac --- .../{databricks.yml.tmpl => databricks.yml} | 12 ++++---- .../recreate_rename_conflict/out.test.toml | 2 +- .../recreate_rename_conflict/output.txt | 13 +++------ .../recreate_rename_conflict/script | 19 ++---------- .../recreate_rename_conflict/test.toml | 6 +++- .../recreate_rename_conflict/databricks.yml | 11 ------- .../recreate_rename_conflict/out.test.toml | 3 -- .../recreate_rename_conflict/output.txt | 29 ------------------- .../recreate_rename_conflict/script | 10 ------- .../recreate_rename_conflict/test.toml | 6 ---- 10 files changed, 19 insertions(+), 92 deletions(-) rename acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/{databricks.yml.tmpl => databricks.yml} (83%) delete mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml delete mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml delete mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt delete mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script delete mode 100644 acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml similarity index 83% rename from acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl rename to acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml index e49a6b4a749..2e56abda01b 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml.tmpl +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: rename-conflict-postgres-synced-table-$UNIQUE_NAME + name: test-bundle sync: paths: [] @@ -7,19 +7,19 @@ sync: resources: schemas: pipeline_storage: - name: pipeline_storage_$UNIQUE_NAME + name: pipeline_storage catalog_name: main comment: "Pipeline storage for the synced-table rename-conflict test" postgres_projects: my_project: - project_id: test-pg-proj-$UNIQUE_NAME + project_id: test-pg-proj display_name: "Test Project for Synced Table Rename Conflict" pg_version: 17 postgres_catalogs: my_catalog: - catalog_id: lakebase_test_$UNIQUE_NAME + catalog_id: lakebase_test branch: ${resources.postgres_projects.my_project.id}/branches/production postgres_database: appdb create_database_if_missing: true @@ -27,7 +27,7 @@ resources: postgres_synced_tables: my_table: synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced - source_table_full_name: main.source_$UNIQUE_NAME.trips_source + source_table_full_name: main.source.trips_source primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb @@ -40,7 +40,7 @@ resources: other_table: synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_other - source_table_full_name: main.source_$UNIQUE_NAME.trips_source + source_table_full_name: main.source.trips_source primary_key_columns: ["tpep_pickup_datetime"] scheduling_policy: SNAPSHOT postgres_database: appdb diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml index 7ad46517eca..ce8be6534d3 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml @@ -1,4 +1,4 @@ -Cloud = true +Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt index 6ef55c4299b..3b294de57fb 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt @@ -1,10 +1,6 @@ -Creating temporary source table: main.source_[UNIQUE_NAME].trips_source -{ - "full_name": "main.source_[UNIQUE_NAME]" -} >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Created postgres_catalogs.my_catalog Created postgres_projects.my_project Created postgres_synced_tables.my_table @@ -17,10 +13,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml public.trips_synced public.trips_other >>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot recreate resources.postgres_synced_tables.my_table: synced table with such id already exists (409 ALREADY_EXISTS) -Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/synced_tables?synced_table_id=lakebase_test_[UNIQUE_NAME].public.trips_other +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/synced_tables?synced_table_id=lakebase_test.public.trips_other HTTP Status: 409 Conflict API error_code: ALREADY_EXISTS API message: synced table with such id already exists @@ -41,7 +37,6 @@ This action will result in the deletion of the following Lakebase projects along all their branches, databases, and endpoints. All data stored in them will be permanently lost: delete resources.postgres_projects.my_project -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/rename-conflict-postgres-synced-table-[UNIQUE_NAME]/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default Destroy: 4 deleted -Cleaning up temporary source table diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script index cde461b74bd..f823d78df9d 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script @@ -1,28 +1,15 @@ -# Per-test source table to avoid the 20-synced-tables-per-source-table limit. -echo "Creating temporary source table: main.source_$UNIQUE_NAME.trips_source" -$CLI schemas create source_$UNIQUE_NAME main -o json | jq '{full_name}' -MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ - \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", - \"statement\": \"CREATE TABLE main.source_$UNIQUE_NAME.trips_source AS SELECT * FROM samples.nyctaxi.trips LIMIT 10\", - \"wait_timeout\": \"45s\" - }" > /dev/null - cleanup() { trace $CLI bundle destroy --auto-approve - echo "Cleaning up temporary source table" - $CLI tables delete main.source_$UNIQUE_NAME.trips_source || true - $CLI schemas delete main.source_$UNIQUE_NAME || true - rm -f "$OUT_REQUESTS" } trap cleanup EXIT -envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy title "Renaming my_table onto other_table's id fails fast (no ALREADY_EXISTS retry)" # Point my_table at other_table's synced_table_id. synced_table_id is a provided-id # field, so the plan recreates my_table: delete the old id, then create other_table's -# id — which a separate resource already owns. The create must surface ALREADY_EXISTS -# immediately rather than retrying for minutes, since no wait can free that name. +# id. The testserver reports ALREADY_EXISTS for the taken id, and the same-id guard +# must surface it immediately rather than retrying for minutes, since no wait can free +# a name a separate resource owns. trace update_file.py databricks.yml "public.trips_synced" "public.trips_other" trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml index b151761c7e0..0d1663c058d 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml @@ -1,4 +1,8 @@ -Cloud = true +# Local only. On cloud a synced-table create with an already-taken id replaces the +# existing table (replace_existing) rather than returning ALREADY_EXISTS, so the +# rename never conflicts there — the same-id guard this exercises is only observable +# against the testserver, which reports the conflict. +Cloud = false # The same-id guard that makes a rename-to-existing fail fast (instead of retrying # the create for minutes) is a direct-engine feature. diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml deleted file mode 100644 index b9c9d8a5e04..00000000000 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/databricks.yml +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: test-bundle - -resources: - vector_search_endpoints: - endpoint_a: - name: vs-endpoint-alpha - endpoint_type: STANDARD - endpoint_b: - name: vs-endpoint-beta - endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml deleted file mode 100644 index 59b56a2037c..00000000000 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt deleted file mode 100644 index 4d581fb4d51..00000000000 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/output.txt +++ /dev/null @@ -1,29 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created vector_search_endpoints.endpoint_a -Created vector_search_endpoints.endpoint_b -Files: 3 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -=== Renaming endpoint_a onto endpoint_b's name fails fast (no ALREADY_EXISTS retry) ->>> update_file.py databricks.yml name: vs-endpoint-alpha name: vs-endpoint-beta - ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot recreate resources.vector_search_endpoints.endpoint_a: Vector search endpoint with name vs-endpoint-beta already exists (409 RESOURCE_ALREADY_EXISTS) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/vector-search/endpoints -HTTP Status: 409 Conflict -API error_code: RESOURCE_ALREADY_EXISTS -API message: Vector search endpoint with name vs-endpoint-beta already exists - -Files: 1 uploaded, 0 deleted - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.vector_search_endpoints.endpoint_b - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 1 deleted diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script deleted file mode 100644 index 5826c1965d8..00000000000 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/script +++ /dev/null @@ -1,10 +0,0 @@ -trace $CLI bundle deploy - -title "Renaming endpoint_a onto endpoint_b's name fails fast (no ALREADY_EXISTS retry)" -# name is a provided-id field, so the plan recreates endpoint_a: delete vs-endpoint-alpha, -# then create vs-endpoint-beta — which endpoint_b already owns. The create must surface -# ALREADY_EXISTS immediately rather than retrying, since no wait can free that name. -trace update_file.py databricks.yml "name: vs-endpoint-alpha" "name: vs-endpoint-beta" -trace musterr $CLI bundle deploy - -trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml deleted file mode 100644 index 8f434449bf5..00000000000 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate_rename_conflict/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -Cloud = false -RecordRequests = false - -# Local-only fast guard. The cloud path for the same-id guard is covered by -# postgres_synced_tables/recreate_rename_conflict. Direct engine only (no VS terraform provider). -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From f95c4ef042c0fd84cf155cf6d11de488504cfbeb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 13:28:41 +0200 Subject: [PATCH 5/6] direct: make the recreate retry code-agnostic; faithful testserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the whole conflict class (ErrResourceConflict) instead of just error_code ALREADY_EXISTS: the lingering-teardown race the retry waits out can surface as either ALREADY_EXISTS (postgres synced tables) or RESOURCE_ALREADY_EXISTS (apps, vector search endpoints), and which code a backend returns should not decide whether we retry. The same-id guard still fails a rename (a provided-id change) fast, so a genuine name collision never spins for minutes regardless of code. The testserver no longer returns ALREADY_EXISTS when creating a synced table whose id already exists — the real backend replaces it rather than conflicting, so the previous rename-conflict test could not reproduce on cloud. Drop that test; the guard's regression coverage is the apps rename and the existing vector_search_endpoints/recreate/create-fails, which now exercise the retry path and would spin without the guard. The destination-table teardown race stays. Co-authored-by: Isaac --- .../recreate_conflict_retry/output.txt | 2 +- .../recreate_rename_conflict/databricks.yml | 52 ------------------- .../recreate_rename_conflict/out.test.toml | 5 -- .../recreate_rename_conflict/output.txt | 42 --------------- .../recreate_rename_conflict/script | 15 ------ .../recreate_rename_conflict/test.toml | 9 ---- bundle/direct/apply.go | 11 ++-- libs/testserver/postgres.go | 8 +-- 8 files changed, 13 insertions(+), 131 deletions(-) delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script delete mode 100644 acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt index 3393a840eae..0f0c47034e2 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate_conflict_retry/output.txt @@ -13,7 +13,7 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Warn: deploying resources.postgres_synced_tables.my_table: Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: Failing setup of Delta sync table: Destination table lakebase_test.public.trips_synced already exists +Warn: deploying resources.postgres_synced_tables.my_table: Create still conflicts; the previous delete is likely still finishing, retrying: Failing setup of Delta sync table: Destination table lakebase_test.public.trips_synced already exists Recreated postgres_synced_tables.my_table Files: 0 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 3 unchanged diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml deleted file mode 100644 index 2e56abda01b..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/databricks.yml +++ /dev/null @@ -1,52 +0,0 @@ -bundle: - name: test-bundle - -sync: - paths: [] - -resources: - schemas: - pipeline_storage: - name: pipeline_storage - catalog_name: main - comment: "Pipeline storage for the synced-table rename-conflict test" - - postgres_projects: - my_project: - project_id: test-pg-proj - display_name: "Test Project for Synced Table Rename Conflict" - pg_version: 17 - - postgres_catalogs: - my_catalog: - catalog_id: lakebase_test - branch: ${resources.postgres_projects.my_project.id}/branches/production - postgres_database: appdb - create_database_if_missing: true - - postgres_synced_tables: - my_table: - synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_synced - source_table_full_name: main.source.trips_source - primary_key_columns: ["tpep_pickup_datetime"] - scheduling_policy: SNAPSHOT - postgres_database: appdb - branch: ${resources.postgres_projects.my_project.id}/branches/production - create_database_objects_if_missing: true - timeseries_key: tpep_pickup_datetime - new_pipeline_spec: - storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} - storage_schema: ${resources.schemas.pipeline_storage.name} - - other_table: - synced_table_id: ${resources.postgres_catalogs.my_catalog.catalog_id}.public.trips_other - source_table_full_name: main.source.trips_source - primary_key_columns: ["tpep_pickup_datetime"] - scheduling_policy: SNAPSHOT - postgres_database: appdb - branch: ${resources.postgres_projects.my_project.id}/branches/production - create_database_objects_if_missing: true - timeseries_key: tpep_pickup_datetime - new_pipeline_spec: - storage_catalog: ${resources.schemas.pipeline_storage.catalog_name} - storage_schema: ${resources.schemas.pipeline_storage.name} diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml deleted file mode 100644 index ce8be6534d3..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Cloud = false -CloudEnvs.azure = false -CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt deleted file mode 100644 index 3b294de57fb..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/output.txt +++ /dev/null @@ -1,42 +0,0 @@ - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created postgres_catalogs.my_catalog -Created postgres_projects.my_project -Created postgres_synced_tables.my_table -Created postgres_synced_tables.other_table -Created schemas.pipeline_storage -Files: 0 uploaded, 0 deleted -Resources: 5 created, 0 changed, 0 deleted, 0 unchanged - -=== Renaming my_table onto other_table's id fails fast (no ALREADY_EXISTS retry) ->>> update_file.py databricks.yml public.trips_synced public.trips_other - ->>> musterr [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Error: cannot recreate resources.postgres_synced_tables.my_table: synced table with such id already exists (409 ALREADY_EXISTS) - -Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/synced_tables?synced_table_id=lakebase_test.public.trips_other -HTTP Status: 409 Conflict -API error_code: ALREADY_EXISTS -API message: synced table with such id already exists - -Files: 0 uploaded, 0 deleted - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.postgres_catalogs.my_catalog - delete resources.postgres_projects.my_project - delete resources.postgres_synced_tables.other_table - delete resources.schemas.pipeline_storage - -This action will result in the deletion of the following UC schemas. Any underlying data may be lost: - delete resources.schemas.pipeline_storage - -This action will result in the deletion of the following Lakebase projects along with -all their branches, databases, and endpoints. All data stored in them will be permanently lost: - delete resources.postgres_projects.my_project - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default - -Destroy: 4 deleted diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script deleted file mode 100644 index f823d78df9d..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/script +++ /dev/null @@ -1,15 +0,0 @@ -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -trace $CLI bundle deploy - -title "Renaming my_table onto other_table's id fails fast (no ALREADY_EXISTS retry)" -# Point my_table at other_table's synced_table_id. synced_table_id is a provided-id -# field, so the plan recreates my_table: delete the old id, then create other_table's -# id. The testserver reports ALREADY_EXISTS for the taken id, and the same-id guard -# must surface it immediately rather than retrying for minutes, since no wait can free -# a name a separate resource owns. -trace update_file.py databricks.yml "public.trips_synced" "public.trips_other" -trace musterr $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml deleted file mode 100644 index 0d1663c058d..00000000000 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate_rename_conflict/test.toml +++ /dev/null @@ -1,9 +0,0 @@ -# Local only. On cloud a synced-table create with an already-taken id replaces the -# existing table (replace_existing) rather than returning ALREADY_EXISTS, so the -# rename never conflicts there — the same-id guard this exercises is only observable -# against the testserver, which reports the conflict. -Cloud = false - -# The same-id guard that makes a rename-to-existing fail fast (instead of retrying -# the create for minutes) is a direct-engine feature. -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 05b13cf54e7..8e560b7124e 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -152,8 +152,11 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // The delete-wait above only observes the resource's own read (e.g. the synced-table // record). A delete can leave *backing* objects behind that it can't see — a synced // table's destination Postgres table is dropped on a slower schedule — so the create - // of the same id can still fail with ALREADY_EXISTS. Retry the create until that - // teardown finishes, then surface the result. + // of the same id can still conflict. Retry the create until that teardown finishes, + // then surface the result. We match the whole conflict class (ErrResourceConflict) + // rather than a single error code: the same lingering-teardown race can surface as + // ALREADY_EXISTS or RESOURCE_ALREADY_EXISTS depending on the resource, and both mean + // the same thing here. // // Only when the recreate re-creates the *same* id, though. If it changed a // provided-id field (e.g. renamed an app, or pointed a synced table at a new @@ -168,8 +171,8 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat switch { case createErr == nil: return &struct{}{}, nil - case !idChanged && errors.Is(createErr, apierr.ErrAlreadyExists): - log.Warnf(ctx, "Create hit ALREADY_EXISTS; the previous delete is likely still finishing, retrying: %s", createErr) + case !idChanged && errors.Is(createErr, apierr.ErrResourceConflict): + log.Warnf(ctx, "Create still conflicts; the previous delete is likely still finishing, retrying: %s", createErr) return nil, retries.Continues("create still conflicts with the deleting resource") default: return nil, retries.Halt(createErr) diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index e552c155db0..369e119760e 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1681,9 +1681,11 @@ func (s *FakeWorkspace) PostgresSyncedTableCreate(req Request, syncedTableID str name := "synced_tables/" + syncedTableID - if _, exists := s.PostgresSyncedTables[name]; exists { - return postgresErrorResponse(409, "ALREADY_EXISTS", "synced table with such id already exists") - } + // Creating a synced table whose id already exists does not conflict on the real + // backend — CreateSyncedTable replaces the existing table — so we overwrite below + // rather than returning ALREADY_EXISTS. The only create conflict the backend + // reports is the transient one below. + // Simulate the destination Postgres table being dropped asynchronously: for a few // creates after a delete, the record is gone from GET but the create still conflicts. if s.postgresSyncedTablesDeleting[name] > 0 { From cf3f2c147a2dae314c744607f5004a85674ba110 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 21 Sep 2026 14:39:35 +0200 Subject: [PATCH 6/6] acc: record requests in the apps rename-conflict test to prove no retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same-id guard should attempt the rename's create exactly once. Record the requests and print the recreate's /apps calls so the golden shows a single POST /api/2.0/apps (create app-beta) after the delete — a retry regression would show it repeated. Co-authored-by: Isaac --- .../apps/recreate_rename_conflict/output.txt | 19 ++++++++++++++++++- .../apps/recreate_rename_conflict/script | 9 ++++++--- .../apps/recreate_rename_conflict/test.toml | 5 ++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt b/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt index 2f5adeb3b6d..2abbcf0d60a 100644 --- a/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/output.txt @@ -6,7 +6,7 @@ Created apps.app_b Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== Renaming app_a onto app_b's name fails fast (no ALREADY_EXISTS retry) +=== Renaming app_a onto app_b's name fails fast (create attempted once, no retry) >>> update_file.py databricks.yml name: app-alpha name: app-beta >>> musterr [CLI] bundle deploy @@ -20,6 +20,23 @@ API message: An app with the same name already exists: app-beta Files: 1 uploaded, 0 deleted +>>> print_requests.py //apps +{ + "method": "DELETE", + "path": "/api/2.0/apps/app-alpha" +} +{ + "method": "POST", + "path": "/api/2.0/apps", + "q": { + "no_compute": "true" + }, + "body": { + "description": "", + "name": "app-beta" + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.apps.app_b diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/script b/acceptance/bundle/resources/apps/recreate_rename_conflict/script index 4b645758dcc..397004ed447 100644 --- a/acceptance/bundle/resources/apps/recreate_rename_conflict/script +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/script @@ -1,10 +1,13 @@ trace $CLI bundle deploy -title "Renaming app_a onto app_b's name fails fast (no ALREADY_EXISTS retry)" +title "Renaming app_a onto app_b's name fails fast (create attempted once, no retry)" # name is a provided-id field, so the plan recreates app_a: delete app-alpha, then -# create app-beta — which app_b already owns. The create must surface ALREADY_EXISTS -# immediately rather than retrying, since no wait can free a name another app owns. +# create app-beta — which app_b already owns. The create must surface the conflict +# immediately. Reset the recorded requests first so print_requests below proves the +# recreate attempts the create exactly once, rather than retrying it in a burst. trace update_file.py databricks.yml "name: app-alpha" "name: app-beta" +rm -f "$OUT_REQUESTS" trace musterr $CLI bundle deploy +trace print_requests.py //apps trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml b/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml index a1405e127c0..a11d93410cf 100644 --- a/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml +++ b/acceptance/bundle/resources/apps/recreate_rename_conflict/test.toml @@ -1,5 +1,8 @@ Cloud = false -RecordRequests = false + +# Record requests so the golden proves the rename's create is attempted once (the +# guard fails fast) rather than retried in a burst. +RecordRequests = true # The same-id guard that makes a rename-to-existing fail fast is a direct-engine feature. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]