diff --git a/.gitignore b/.gitignore index b792ef5a..a798543d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ results regression_output tmp_check -spock_create_subscriber +/utils/spock_create_subscriber/spock_create_subscriber +/utils/spock_create_subscriber/.deps/ .vimrc *.o *.so diff --git a/Makefile b/Makefile index 1105a4fb..a4c63dbe 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ EXTENSION = spock PGFILEDESC = "spock - multi-master replication" MODULES = spock_output +SUBDIRS = utils/spock_create_subscriber # Lookup source directory vpath % src src/compat/$(PGVER) diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md new file mode 100644 index 00000000..64528071 --- /dev/null +++ b/docs/creating_subscriber_nodes.md @@ -0,0 +1,38 @@ +## Creating a Subscriber Node with pg_basebackup + +Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. + +### Synopsis: + + `spock_create_subscriber [OPTION]...` + +**Options** + +Specify the following options as needed. + +| Option | Description +|----------|------------- +| `-D`, `--pgdata=DIRECTORY` | The `data` directory to be used for new node. This can be either an empty/non-existing directory, or a directory populated using the `pg_basebackup -X stream` command. +| `--databases` | An optional list of databases to replicate. +| `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. +| `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. +| `--provider-dsn=CONNSTR` | A connection string to the provider. +| `--replication-sets=SETS` | A comma-separated list of replication set names. +| `--apply-delay=DELAY` | The apply delay in seconds (by default 0). +| `--drop-slot-if-exists` | Drop replication slot of conflicting name. +| `-s`, `--stop` | Stop the server once the initialization is done. +| `-v` | Increase logging verbosity. +| `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` +| `--text-types` | Transfer all column values as text rather than binary during initial sync. Use this when provider and subscriber differ in endianness or type representation. + +**Configuration files overrides** + +You can use the following options to override the location of the configuration files. + +| Option | Description +|----------|------------- +|`--hba-conf` | path to the new `pg_hba.conf` +| `--postgresql-conf` | path to the new `postgresql.conf` +| `--recovery-conf` | path to the template recovery configuration + +Unlike `spock.sub_create`'s other data sync options, this method of cloning ignores replication sets and copies all tables on all databases. However, it's often much faster, especially over high-bandwidth connections. diff --git a/src/spock_fe.c b/src/spock_fe.c index ebe62a82..0a6d66e0 100644 --- a/src/spock_fe.c +++ b/src/spock_fe.c @@ -226,10 +226,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -238,6 +240,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) diff --git a/tests/tap/schedule b/tests/tap/schedule index 7e8c457e..a7af339a 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,6 +58,15 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts +test: 048_bidir_join +test: 050_bidir_join_crash_midcatchup +test: 051_bidir_join_multipeer_lag +test: 052_bidir_join_big_txn_liveness +test: 053_bidir_join_unbounded_peer_slot_wal +test: 054_bidir_join_precondition_negatives +test: 055_bidir_join_lww_conflict +test: 056_bidir_join_concurrent_load +test: 057_bidir_join_stall_abort # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # @@ -65,3 +74,4 @@ test: 045_lsn_from_commit_ts # Regression tests test: 103_manager_worker_dboid_race test: 105_sub_disable_retransmit_after_disconnect + diff --git a/tests/tap/schedule-nightly b/tests/tap/schedule-nightly index c193644e..8a48b88c 100644 --- a/tests/tap/schedule-nightly +++ b/tests/tap/schedule-nightly @@ -21,3 +21,10 @@ test: 017_zodan_3n_timeout # deadlock. test: 035_deadlock_retry test: 036_real_deadlock_retry + +# --bidirectional join under sustained OLTP write load (pgedge-loadgen). +# Not yet enabled here: needs its real-world runtime at +# SPOCK_BIDIR_LOAD_LEVEL=medium/heavy characterized on CI hardware first, +# and CI needs a pgedge-loadgen checkout (PGEDGE_LOADGEN_BIN) available -- +# it skip_alls cleanly without one. ~161s locally at the light default. +#test: 049_bidir_join_under_load diff --git a/tests/tap/t/048_bidir_join.pl b/tests/tap/t/048_bidir_join.pl new file mode 100644 index 00000000..770a93de --- /dev/null +++ b/tests/tap/t/048_bidir_join.pl @@ -0,0 +1,759 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 048_bidir_join.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the bidirectional node-join procedure end to end: physical +# backup, recovery to a restore point, catalog strip (capture + origin drop +# + guarded DROP EXTENSION), replication-set/table/sequence restore, and +# catchup (disabled-first subscription to the source, disabled placeholder +# subscriptions to every peer, and a wait for n3 to reach a target LSN +# captured on the source) -- stopping short of enabling any direct peer +# subscription (a later step). +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node +# cluster from create_cluster/cross_wire) +# n3 a real third PostgreSQL instance built via +# `spock_create_subscriber --bidirectional`, physically backed +# up from n1. +# +# Test count breakdown: +# 1 binary found +# 5 create_cluster(2) +# 1 cross_wire n1<->n2 +# 1 custom replication set created on n1 +# 1 table with row_filter added to custom set on n1 +# 1 table with explicit column list added to custom set on n1 +# 1 sequence added to custom set on n1 +# 1 sequence advanced past its initial value on n1 (setval fidelity check) +# 1 partitioned table (parent + 2 children) added to custom set on n1 +# 1 sequence with apostrophe in name added to custom set on n1 +# 1 peer-forwarding test table created on n1 +# 1 peer-forwarding test table replicated to n2 +# 1 --bidirectional exits 0 +# 1 n3 postgres is running +# 1 spock extension installed cleanly on n3 (exactly one row) +# 1 n3 has exactly the catchup and peer origins, none leftover from the basebackup +# 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 +# 1 spock.readonly is lifted on n3 once the join is fully verified +# 1 custom replication set restored on n3 with correct flags +# 1 table membership restored with correct row_filter +# 1 table membership restored with correct explicit column list +# 1 sequence value restored exactly (last_value) +# 1 sequence is_called restored exactly +# 1 sequence pr3_test_seq is a member of pr3_test_repset on n3 +# 1 partitioned table parent + 2 children all present in repset on n3 +# 1 apostrophe-named sequence value restored on n3 +# 1 apostrophe-named sequence is_called restored on n3 +# 1 apostrophe-named sequence is a member of pr3_test_repset on n3 +# 1 manifest: source_slot_name populated +# 1 manifest: source_restore_lsn populated +# 1 manifest: node_dsn populated +# 1 source slot exists on n1 +# 1 catchup subscription sub_n3_n1 is replicating on n3 +# 1 forwarding cleared on sub_n3_n1 after cutover +# 1 direct peer subscription sub_n3_n2 is replicating on n3 after cutover +# 1 peer slot created on n2 during the coverage barrier +# 1 n2's post-cutover write reached n3 via the direct sub_n3_n2 path +# 1 n3's origin for peer n2 advanced via the direct subscription +# 1 reverse subscription sub_n2_n3 is replicating on n2 +# 1 reverse subscription sub_n1_n3 is replicating on n1 +# 1 n3's post-join write reached the source via sub_n1_n3 +# 1 n3's post-join write reached the peer via sub_n2_n3 +# 1 --cleanup --force exits 0 +# 1 source slot removed from n1 after cleanup +# 1 n3 data directory removed after cleanup --force +# 1 manifest removed after cleanup +# 1 --bidirectional rejects a multi-database request +# 1 --max-wait rejects a value that overflows int +# 1 --stall-timeout rejects a non-numeric value +# 1 --apply-delay rejects a value that overflows int +# 1 --bidirectional aborts when another database on the source has spock configured +# 1 --bidirectional rejects a broken full-mesh topology (disabled subscription) +# 1 --bidirectional rejects mismatched replication-set flags between source and peer +# 1 a broken --extra-basebackup-args makes the base backup fail +# 1 pending-cleanup sidecar written before the failed backup +# 1 pending-cleanup sidecar is mode 0600 +# 1 source slot still exists on n1 after the failed backup (orphaned) +# 1 --cleanup --force recovers via the pending sidecar +# 1 source slot removed from n1 via sidecar-based cleanup +# 1 pending-cleanup sidecar removed after cleanup +# 1 a broken backup orphans a slot for the retry-cleanup test +# 1 pending sidecar written for the retry-cleanup test +# 1 --cleanup exits non-zero when the source is unreachable +# 1 pending sidecar retained after an incomplete cleanup +# 1 n1 postgres is running again +# 1 --cleanup --force succeeds once the source is reachable again +# 1 pending sidecar removed once cleanup actually completed +# 1 destroy_cluster +# --- +# 72 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 72; +use File::Path qw(remove_tree); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +my $n1_sysid = scalar_query(1, "SELECT system_identifier FROM pg_control_system()"); + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# Seed n1 with a custom replication set, a table with a row_filter, and a +# sequence, to exercise the catalog capture/restore with non-default state +# rather than just the three built-in sets. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_create('pr3_test_repset', true, true, true, false)"; +pass('custom replication set created on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_tbl (id serial primary key, region text, value integer)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_tbl', synchronize_data := false, " . + "row_filter := 'region = ''east''')"; +pass('table with row_filter added to custom set on n1'); + +# Table with an explicit, non-default column list, to exercise the +# columns := restore path (captured/restored as a bare array- +# literal string relying on implicit text[] coercion) -- previously +# untested, so a round-trip regression here could pass silently. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_cols (id serial primary key, region text, " . + "value integer, secret text)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_cols', synchronize_data := false, " . + "columns := ARRAY['id', 'region', 'value'])"; +pass('table with explicit column list added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE pr3_test_seq"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_seq('pr3_test_repset', 'pr3_test_seq')"; +pass('sequence added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT setval('pr3_test_seq', 42, true)"; +my $seq_before = scalar_query(1, "SELECT last_value FROM pr3_test_seq"); +is($seq_before, '42', 'sequence advanced past its initial value on n1'); + +# Partitioned table: parent + 2 children get separate captured membership +# rows (that's how include_partitions => true populated them here); restore +# must not try to re-add children a second time via the parent's own +# include_partitions => true call. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part (id int, region text, PRIMARY KEY (id, region)) PARTITION BY LIST (region)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_east PARTITION OF pr3_test_part FOR VALUES IN ('east')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_west PARTITION OF pr3_test_part FOR VALUES IN ('west')"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_part', synchronize_data := false, " . + "include_partitions := true)"; +pass('partitioned table (parent + 2 children) added to custom set on n1'); + +# Sequence with an apostrophe in its name, to exercise setval() quoting. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(CREATE SEQUENCE "weird's_seq"); +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT spock.repset_add_seq('pr3_test_repset', '"weird''s_seq"')); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT setval('"weird''s_seq"', 7, true)); +pass('sequence with apostrophe in name added to custom set on n1'); + +# Table used later to verify n3's origin for peer n2 advances via forwarding. +# Created on n1 only and left to arrive on n2 via DDL replication (creating +# it directly on both sides races the already-established cross-wire DDL +# replay); spock.include_ddl_repset=on adds it to 'default' on each node +# once it lands there. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr4_peer_tbl (id serial primary key, val text)"; +pass('peer-forwarding test table created on n1'); + +my $tbl_on_n2 = '0'; +for (1 .. 15) { + $tbl_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'pr4_peer_tbl'"); + last if $tbl_on_n2 eq '1'; + sleep(1); +} +is($tbl_on_n2, '1', 'peer-forwarding test table replicated to n2'); + +# check_preconditions() requires all of n1's outbound replication to have +# caught up (no unreplicated DDL/data still in flight to n2); wait for the +# setup above to drain. +for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + last if defined $lag && $lag eq '0'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional continues through physical backup / catalog strip / +# repset restore, stopping before the catchup subscription. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3'; +my $manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; + +# n3's postgresql.conf is copied verbatim from n1 by the basebackup, port and +# all -- since all nodes run on the same host in this test, n3 must be given +# an override with its own port (a real cross-host join wouldn't need this). +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + ], + '--bidirectional exits 0' +); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); + +my $ext_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_extension WHERE extname = 'spock'"`; +$ext_count =~ s/\s+//g; +is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); + +# By this point the catchup subscription and the one disabled peer +# subscription (n2) have each created their own origin -- exactly 2, not +# more. Anything beyond that would mean an origin survived from the +# basebackup instead of being dropped by the catalog strip. +my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; +$origin_count =~ s/\s+//g; +is($origin_count, '2', + 'n3 has exactly the catchup and peer origins, none leftover from the basebackup'); + +my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; +$n3_sysid =~ s/\s+//g; +isnt($n3_sysid, $n1_sysid, + 'n3 was given its own system identifier (pg_resetwal), distinct from n1'); + +my $readonly = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SHOW spock.readonly"`; +$readonly =~ s/\s+//g; +is($readonly, 'off', + "spock.readonly is lifted on n3 once the join is fully verified"); + +my $repset_flags = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT replicate_insert, replicate_update, replicate_delete, replicate_truncate FROM spock.replication_set WHERE set_name = 'pr3_test_repset'"`; +$repset_flags =~ s/\s+//g; +is($repset_flags, 't|t|t|f', 'custom replication set restored on n3 with correct flags'); + +my $row_filter = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT pg_get_expr(rts.set_row_filter, rts.set_reloid) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset'"`; +$row_filter =~ s/^\s+|\s+$//g; +like($row_filter, qr/region\s*=\s*'east'/, 'table membership restored with correct row_filter'); + +my $columns = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT rts.set_att_list FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text = 'pr3_test_cols'"`; +$columns =~ s/^\s+|\s+$//g; +is($columns, '{id,region,value}', + 'table membership restored with correct explicit column list'); + +my $seq_last_value = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT last_value FROM pr3_test_seq"`; +$seq_last_value =~ s/\s+//g; +is($seq_last_value, '42', 'sequence value restored exactly (last_value)'); + +my $seq_is_called = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT is_called FROM pr3_test_seq"`; +$seq_is_called =~ s/\s+//g; +is($seq_is_called, 't', 'sequence is_called restored exactly'); + +# pr3_test_seq must be an actual member of pr3_test_repset on n3, not just +# have its value restored (a regression here is the sequence-membership bug: +# setval() alone leaves the sequence unpublished). +my $seq_member = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = 'pr3_test_seq'"`; +$seq_member =~ s/\s+//g; +is($seq_member, '1', 'sequence pr3_test_seq is a member of pr3_test_repset on n3'); + +# Partitioned table: parent + 2 children must all be present as distinct +# memberships (a regression here is include_partitions => true re-adding +# already-captured children and violating the (set_id, set_reloid) PK, +# which would have aborted the join above rather than just miscounting). +my $part_member_count = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text LIKE 'pr3_test_part%'"`; +$part_member_count =~ s/\s+//g; +is($part_member_count, '3', 'partitioned table parent + 2 children all present in repset on n3'); + +# Sequence with an apostrophe in its name: value/is_called restored and +# membership present, without a SQL syntax error breaking the whole run. +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out =~ s/^\s+|\s+$//g if defined $out; + return $out; +} + +my $weird_seq_value = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT last_value FROM "weird\'s_seq"'); +is($weird_seq_value, '7', "apostrophe-named sequence value restored on n3"); + +my $weird_seq_called = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT is_called FROM "weird\'s_seq"'); +is($weird_seq_called, 't', "apostrophe-named sequence is_called restored on n3"); + +my $weird_seq_member = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = '\"weird''s_seq\"'"); +is($weird_seq_member, '1', "apostrophe-named sequence is a member of pr3_test_repset on n3"); + +# ============================================================================= +# Manifest content checks +# ============================================================================= +my $manifest_content = ''; +if (-f $manifest) { + open my $fh, '<', $manifest or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +ok($manifest_content =~ /"source_slot_name":\s*"[^"]+"/, + 'manifest: source_slot_name populated'); +ok($manifest_content =~ /"source_restore_lsn":\s*"[0-9A-Fa-f]+\/[0-9A-Fa-f]+"/, + 'manifest: source_restore_lsn populated'); +ok($manifest_content =~ /"node_dsn":\s*"[^"]+"/, + 'manifest: node_dsn populated'); + +my $source_slot_exists = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($source_slot_exists >= 1, 'source slot exists on n1'); + +# ============================================================================= +# TEST: catchup subscription and, after cutover, the direct peer +# subscription are both replicating; a post-cutover write on n2 reaches n3 +# via the direct path, advancing n3's origin for n2. +# ============================================================================= +my $sub_status = ''; +for (1 .. 30) { + $sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n1')"); + last if $sub_status eq 'replicating'; + sleep(1); +} +is($sub_status, 'replicating', 'catchup subscription sub_n3_n1 is replicating on n3'); + +is(psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT forward_origins FROM spock.sub_show_status('sub_n3_n1')"), + '', "forwarding cleared on sub_n3_n1 after cutover"); + +my $peer_sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n2')"); +is($peer_sub_status, 'replicating', + 'direct peer subscription sub_n3_n2 is replicating on n3 after cutover'); + +# Origin name matches what create_disabled_peer_subscriptions() computed for +# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')) -- the same +# value is also the slot name create_peer_slot() created on n2 during the +# coverage barrier. +my $n2_origin_name = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); + +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name = '$n2_origin_name'"), + '1', 'peer slot created on n2 during the coverage barrier'); + +my $n2_origin_query = + "SELECT COALESCE(s.remote_lsn::text, '0/0') FROM pg_replication_origin o " . + "LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id " . + "WHERE o.roname = '$n2_origin_name'"; + +# Write on n2 after cutover; forwarding is off and the direct sub_n3_n2 is +# enabled, so this reaches n3 directly from n2, not via n1. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (val) VALUES ('from_n2_post_join')"; + +my $row_on_n3 = '0'; +for (1 .. 30) { + $row_on_n3 = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n2_post_join'"); + last if $row_on_n3 eq '1'; + sleep(1); +} +is($row_on_n3, '1', "n2's post-cutover write reached n3 via the direct sub_n3_n2 path"); + +my $n2_origin_lsn = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced via the direct subscription"); + +# ============================================================================= +# TEST: reverse subscriptions are replicating, and a write on n3 reaches +# both the source and the peer through them -- an external proof, +# independent of the utility's own internal verify_bidirectional_dataflow() +# check. +# ============================================================================= +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n2_n3')"), + 'replicating', 'reverse subscription sub_n2_n3 is replicating on n2'); + +is(psql_capture('-p', $node_ports->[0], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n1_n3')"), + 'replicating', 'reverse subscription sub_n1_n3 is replicating on n1'); + +# Explicit id: pr4_peer_tbl's serial sequence isn't part of the custom +# repset that gets its value round-tripped onto n3 (only pr3_test_seq and +# the apostrophe-named sequence are), so n3's own local copy of the +# sequence is still at its basebackup-time value and would collide with +# the id the n2-post-join row already claimed via replication. +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (id, val) VALUES (1000, 'from_n3_post_join')"; + +my $row_on_n1 = '0'; +for (1 .. 30) { + $row_on_n1 = scalar_query(1, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n1 eq '1'; + sleep(1); +} +is($row_on_n1, '1', "n3's post-join write reached the source via sub_n1_n3"); + +my $row_on_n2_from_n3 = '0'; +for (1 .. 30) { + $row_on_n2_from_n3 = scalar_query(2, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n2_from_n3 eq '1'; + sleep(1); +} +is($row_on_n2_from_n3, '1', "n3's post-join write reached the peer via sub_n2_n3"); + +# ============================================================================= +# TEST: --cleanup --force removes source slot, data directory, and manifest +# ============================================================================= +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $n3_datadir, + ], + '--cleanup --force exits 0' +); + +my $source_slot_after = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); +ok(!-f $manifest, 'manifest removed after cleanup'); + +# ============================================================================= +# TEST: --bidirectional hard-rejects a multi-database request outright, +# rather than silently joining only the first-named database -- all join +# state is per-database. +# ============================================================================= +my $multidb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_multidb'; +remove_tree($multidb_datadir) if -d $multidb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $multidb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--databases', "$dbname,postgres"), + '--bidirectional rejects a multi-database request'); +remove_tree($multidb_datadir) if -d $multidb_datadir; + +# ============================================================================= +# TEST: option parsing rejects malformed/out-of-range integer arguments +# instead of silently truncating them -- e.g. a value like 4294967296 +# wraps to 0 when cast to int after strtol(), which would otherwise turn +# --max-wait=4294967296 into an accepted, effectively-unbounded wait. +# ============================================================================= +my $intcheck_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_intcheck'; +remove_tree($intcheck_datadir) if -d $intcheck_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--max-wait', '4294967296'), + '--max-wait rejects a value that overflows int'); + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--stall-timeout', 'abc'), + '--stall-timeout rejects a non-numeric value'); + +ok(!system_maybe($SCS_BIN, + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--apply-delay', '99999999999999999999'), + '--apply-delay rejects a value that overflows int'); +remove_tree($intcheck_datadir) if -d $intcheck_datadir; + +# ============================================================================= +# TEST: --bidirectional aborts if the source instance has spock configured +# on another database too, even though that database was never named via +# --databases (check_single_spock_database() must fail closed). +# ============================================================================= +system_or_bail "$pg_bin/createdb", '-p', $node_ports->[0], 'pr3_other_db'; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "CREATE EXTENSION spock"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_create('pr3_other_node', 'dbname=pr3_other_db')"; + +my $otherdb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_otherdb'; +remove_tree($otherdb_datadir) if -d $otherdb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $otherdb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional aborts when another database on the source has spock configured'); +remove_tree($otherdb_datadir) if -d $otherdb_datadir; + +system_maybe "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_drop('pr3_other_node')"; +system_maybe "$pg_bin/dropdb", '-p', $node_ports->[0], 'pr3_other_db'; + +# ============================================================================= +# TEST: --bidirectional rejects a broken full-mesh topology -- a disabled +# subscription is not a valid mesh edge, even though it still exists. A +# plain subscription COUNT would not catch this. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_disable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 'f'; + sleep(1); +} + +my $mesh_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_mesh'; +remove_tree($mesh_datadir) if -d $mesh_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $mesh_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects a broken full-mesh topology (disabled subscription)'); +remove_tree($mesh_datadir) if -d $mesh_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_enable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 't'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional rejects mismatched replication-set definitions for a +# selected (subscription-referenced) set between source and peer -- a +# repset the forwarding path and a future direct-peer path disagree on can +# permanently drop changes on cutover. DDL replication is disabled for the +# ALTER itself so the mismatch is real and local to n2. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := false)"; + +my $repset_mismatch_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_repset_mismatch'; +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $repset_mismatch_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects mismatched replication-set flags between source and peer'); +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := true)"; + +# ============================================================================= +# TEST: a failed base backup leaves the source slot recoverable via +# --cleanup, even though the real manifest was never written -- a +# pending-cleanup sidecar is persisted right after source slot creation, +# before the backup even starts, since data_dir must stay empty until +# pg_basebackup runs. +# ============================================================================= +my $failed_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_failed'; +my $pending_sidecar = "${failed_datadir}.spock_bidir_pending.json"; +remove_tree($failed_datadir) if -d $failed_datadir; +unlink($pending_sidecar) if -f $pending_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $failed_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_xyz'), + 'a broken --extra-basebackup-args makes the base backup fail'); + +ok(-f $pending_sidecar, 'pending-cleanup sidecar written before the failed backup'); + +my $sidecar_mode = (stat($pending_sidecar))[2] & 07777; +is(sprintf('%04o', $sidecar_mode), '0600', + 'pending-cleanup sidecar is mode 0600 (may carry a DSN password)'); + +my $slot_after_failed_backup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($slot_after_failed_backup >= 1, + 'source slot still exists on n1 after the failed backup (orphaned)'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $failed_datadir, + ], + '--cleanup --force recovers via the pending sidecar (no real manifest exists)' +); + +my $slot_after_sidecar_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($slot_after_sidecar_cleanup, '0', + 'source slot removed from n1 via sidecar-based cleanup'); + +ok(!-f $pending_sidecar, 'pending-cleanup sidecar removed after cleanup'); +remove_tree($failed_datadir) if -d $failed_datadir; + +# ============================================================================= +# TEST: an incomplete cleanup (source unreachable) exits non-zero and keeps +# the pending sidecar so it can be retried, instead of unconditionally +# deleting the only retry record. +# ============================================================================= +my $retry_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_retry'; +my $retry_sidecar = "${retry_datadir}.spock_bidir_pending.json"; +remove_tree($retry_datadir) if -d $retry_datadir; +unlink($retry_sidecar) if -f $retry_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $retry_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_retry'), + 'a broken backup orphans a slot for the retry-cleanup test'); +ok(-f $retry_sidecar, 'pending sidecar written for the retry-cleanup test'); + +my $n1_datadir = $config->{node_datadirs}->[0]; +system_or_bail "$pg_bin/pg_ctl", 'stop', '-D', $n1_datadir, '-m', 'fast'; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir), + '--cleanup exits non-zero when the source is unreachable'); +ok(-f $retry_sidecar, + 'pending sidecar retained after an incomplete cleanup (retryable)'); + +system_or_bail "$pg_bin/pg_ctl", 'start', '-D', $n1_datadir, + '-l', "$config->{log_dir}/n1_retry_restart.log"; +ok(wait_for_pg_ready($host, $node_ports->[0], $pg_bin, 30), + 'n1 postgres is running again'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir, + ], + '--cleanup --force succeeds once the source is reachable again' +); +ok(!-f $retry_sidecar, + 'pending sidecar removed once cleanup actually completed'); +remove_tree($retry_datadir) if -d $retry_datadir; + +# ============================================================================= +# CLEANUP +# ============================================================================= +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_tbl"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_part"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_cols"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP SEQUENCE IF EXISTS pr3_test_seq"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(DROP SEQUENCE IF EXISTS "weird's_seq"); +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_drop('pr3_test_repset')"; +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Cleanup'); diff --git a/tests/tap/t/050_bidir_join_crash_midcatchup.pl b/tests/tap/t/050_bidir_join_crash_midcatchup.pl new file mode 100644 index 00000000..9edb4abe --- /dev/null +++ b/tests/tap/t/050_bidir_join_crash_midcatchup.pl @@ -0,0 +1,392 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 050_bidir_join_crash_midcatchup.pl - spock_create_subscriber +# --bidirectional survives the +# utility process itself being +# killed mid-catchup +# ============================================================================= +# 048_bidir_join.pl and 049_bidir_join_under_load.pl both prove --cleanup +# --force recovers a join that failed during the *base backup* phase (a +# deliberately broken --extra-basebackup-args). Neither kills the utility +# process itself once it is past that point and into the catchup replay -- +# the phase this session's investigation (spurious SUB_DISABLE / apply-idle- +# timeout misclassification, PR 607) was all about. The design doc's own +# resumability contract for a mid-catchup crash is "not a resumable journal": +# the practical recovery is --cleanup --force + a fresh basebackup, same as +# for a backup failure -- but that path was never actually exercised for a +# crash landing *after* the source slot exists and the catchup subscription +# is already replicating, only for one landing *before* the backup completes. +# +# This test seeds n1 with enough rows that the catchup replay is guaranteed +# to still be in progress a few seconds in, starts --bidirectional in the +# background, waits until n3's catchup subscription (sub_n3_n1) is +# genuinely mid-replay (present, replicating, and only partially caught up), +# then SIGKILLs the utility process outright -- no SIGTERM, no chance for +# graceful shutdown -- and verifies: --cleanup --force still recovers +# cleanly (source slot removed, data directory removed); the pre-existing +# n1<->n2 mesh is unaffected; and a fresh --bidirectional retry against the +# same source afterward succeeds end-to-end with correct data, proving the +# cluster is genuinely left in a rejoinable state, not just "cleaned up". +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out //= ''; + $out =~ s/^\s+|\s+$//g; + return $out; +} + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +# Poll a log file's content for a pattern (rather than a fixed offset read) +# since the file may not exist yet when polling starts. +sub wait_for_log_pattern { + my ($logfile, $pattern, $timeout) = @_; + for (1 .. $timeout) { + if (-f $logfile) { + open(my $fh, '<', $logfile) or die "Cannot open $logfile: $!"; + local $/; + my $content = <$fh>; + close($fh); + return 1 if defined $content && $content =~ $pattern; + } + sleep(1); + } + return 0; +} + +# ============================================================================= +# SETUP: a continuous writer on n1, so catchup always has a growing WAL +# delta to replay. --bidirectional does a *physical* basebackup first -- +# a one-time seed inserted before the join is already present in that +# backup, leaving nothing for the logical catchup subscription to actually +# replay (confirmed empirically: a 400k-row one-time seed made catchup +# resolve instantly, giving no window to interrupt at all). Only rows +# written *during* the join, after the backup's start LSN, are real +# catchup work. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE crash_test_tbl (id bigint PRIMARY KEY, val text)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE crash_test_id_seq"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', q{ + CREATE PROCEDURE crash_test_load(n_batches int, batch_rows int) + LANGUAGE plpgsql AS $$ + DECLARE i int; + BEGIN + FOR i IN 1..n_batches LOOP + INSERT INTO crash_test_tbl + SELECT nextval('crash_test_id_seq'), 'x' || g + FROM generate_series(1, batch_rows) g; + COMMIT; + PERFORM pg_sleep(0.02); + END LOOP; + END $$; +}; +pass('crash_test_tbl and writer procedure created on n1'); + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1->n2 replication never drained after setup; cannot proceed'); + +# The writer is started only after --bidirectional's own +# check_preconditions() has already passed (see wait_for_slot_created() +# below) -- it hard-rejects up front if the source has *any* unreplicated +# changes pending to an existing peer, so starting it any earlier makes +# every run fail before n3's postgres even comes up (confirmed +# empirically). Kept running through the rest of the physical backup and +# into catchup, then stopped (server-side) once the kill has landed. Same +# technique as 046_apply_worker_exception_misclassification.pl and +# 051_bidir_join_multipeer_lag.pl: killing the client process alone does +# not reliably stop a backend mid-CALL. +my $writer_pid; + +sub start_writer { + $writer_pid = spawn_background("$log_dir/crash_test_writer.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, + '-c', "CALL crash_test_load(100000, 3000)"); +} + +sub stop_writer { + system_or_bail "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " . + "WHERE query LIKE 'CALL crash_test_load%' AND pid <> pg_backend_pid()"; + for (1 .. 30) { + my $still_running = scalar_query(1, + "SELECT count(*) FROM pg_stat_activity WHERE query LIKE 'CALL crash_test_load%'"); + last if defined $still_running && $still_running eq '0'; + sleep(1); + } + kill('TERM', $writer_pid); + waitpid($writer_pid, 0); +} + +# ============================================================================= +# TEST: start --bidirectional, kill it mid-catchup, confirm --cleanup +# --force recovers cleanly +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_crash'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +# The pending-cleanup sidecar is written immediately after the source slot +# is created, i.e. immediately after check_preconditions() has already +# passed -- polling for it (or the real manifest, in case it lands between +# polls) is the same technique 049_bidir_join_under_load.pl uses to know +# it is now safe to resume write load against the source. +sub wait_for_slot_created { + my ($timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $n3_pending || -f $n3_manifest; + sleep(1); + } + return 0; +} + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.crash'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_crash_midcatchup.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '600', +); + +ok(wait_for_slot_created(30), 'source slot created (safe to start the writer)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); +start_writer(); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 60), + 'n3 postgres is running (pre-kill)') + or BAIL_OUT('n3 postgres never came up; see ' . $scs_log); + +# Wait until spock_create_subscriber's own log shows it has entered the +# catchup wait -- sub_n3_n1 exists, is enabled, and wait_for_catchup() is +# now polling its progress against a fixed target LSN captured moments +# earlier. The writer has been running since the source slot was created +# (well before this point: the physical backup transfer, restore-point +# dance, and n3 postgres restart all happen in between), so the target +# captured is already a large, real backlog -- no margin sleep here on +# purpose: with an idle peer (n2) the coverage barrier and clear_forwarding +# phases that follow catchup are fast, so killing as early as possible +# (the instant this line appears) rather than after any extra delay is +# what keeps this test inside the catchup wait specifically, rather than +# racing into later phases. +ok(wait_for_log_pattern($scs_log, qr/Waiting for catchup to the source/, 120), + 'spock_create_subscriber reached the catchup wait phase') + or BAIL_OUT('never reached the catchup wait phase; see ' . $scs_log); + +# The harshest interruption: no SIGTERM, no chance to run any cleanup path +# at all -- a genuine crash, not a graceful stop. +kill('KILL', $scs_pid); +waitpid($scs_pid, 0); +pass('spock_create_subscriber process killed (SIGKILL) mid-catchup'); + +stop_writer(); + +my $final_count = scalar_query(1, "SELECT COUNT(*) FROM crash_test_tbl"); +ok(defined $final_count && $final_count > 0, + "writer produced $final_count rows on n1 before being stopped"); + +my $source_slot_after_kill = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($source_slot_after_kill >= 1, + 'source slot on n1 survives the kill (orphaned, as expected)'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force exits 0 after a mid-catchup kill' +); + +my $source_slot_after_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after_cleanup, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); + +ok(!wait_for_pg_ready($host, $n3_port, $pg_bin, 5), + 'n3 postgres is no longer reachable after cleanup'); + +my $n1n2_ok_after_kill = wait_for_sub_status(1, 'sub_n1_n2', 'replicating', 30) + && wait_for_sub_status(2, 'sub_n2_n1', 'replicating', 30); +ok($n1n2_ok_after_kill, + 'pre-existing n1 <-> n2 mesh is unaffected by the crash + cleanup cycle'); + +# ============================================================================= +# TEST: a fresh --bidirectional retry against the same source succeeds -- +# the cluster is genuinely rejoinable, not just "cleaned up". +# ============================================================================= +ok(wait_for_zero_lag(1, 60), 'replication drained before the retry') + or BAIL_OUT('n1->n2 replication never drained after cleanup; cannot retry'); + +my $scs_log_retry = "$log_dir/scs_crash_midcatchup_retry.log"; +unlink($scs_log_retry) if -f $scs_log_retry; + +my $scs_pid_retry = spawn_background($scs_log_retry, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '600', +); + +my $scs_rc_retry = wait_for_pid($scs_pid_retry, 300); +unless (defined $scs_rc_retry) { + diag("spock_create_subscriber (retry) did not exit within 300s; killing it"); + kill('TERM', $scs_pid_retry); + waitpid($scs_pid_retry, 0); + $scs_rc_retry = -1; +} +is($scs_rc_retry, 0, 'fresh --bidirectional retry after the crash exits 0') + or diag("see $scs_log_retry"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running (retry)'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3 (retry)'); + +my $n3_count_retry = '-1'; +for (1 .. 60) { + $n3_count_retry = scalar_query(3, "SELECT COUNT(*) FROM crash_test_tbl"); + last if $n3_count_retry eq $final_count; + sleep(1); +} +is($n3_count_retry, $final_count, 'all rows present on n3 after the retry'); + +my $hash_n1 = scalar_query(1, + "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) FROM crash_test_tbl x"); +my $hash_n3 = scalar_query(3, + "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) FROM crash_test_tbl x"); +is($hash_n3, $hash_n1, 'content hash matches between n1 and n3 after the retry'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force (final teardown) exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after final cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/tests/tap/t/051_bidir_join_multipeer_lag.pl b/tests/tap/t/051_bidir_join_multipeer_lag.pl new file mode 100644 index 00000000..e0576e54 --- /dev/null +++ b/tests/tap/t/051_bidir_join_multipeer_lag.pl @@ -0,0 +1,414 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 051_bidir_join_multipeer_lag.pl - spock_create_subscriber +# --bidirectional's coverage barrier +# genuinely waits for EVERY existing +# peer, not just the fastest one +# ============================================================================= +# establish_peer_coverage_barrier() (spock_create_subscriber.c) has two +# steps: Hop 1, per peer P, waits for a sync_event emitted on P to land on +# the source (n1); Hop 2, once, fires only after every peer's Hop 1 has +# landed, and cuts the new node over to direct replication. Both +# 048_bidir_join.pl and 049_bidir_join_under_load.pl only ever join a new +# node into a 2-node mesh (n1 + one peer, n2) -- with exactly one peer, the +# "wait for EVERY peer" condition in Hop 2 is vacuously true on every run. A +# bug that fired Hop 2 as soon as *any* single peer's Hop 1 landed, instead +# of *all* of them, would not be caught by either test. +# +# This test builds a genuine 3-node pre-existing mesh (n1, n2, n3, fully +# cross-wired) and keeps n3 under sustained write load throughout the join +# of a 4th node (n4, via n1 as source) -- so n1's replay of n3's changes is +# backlogged right at the moment the coverage barrier runs, while n2 (idle) +# clears its Hop 1 almost immediately. If Hop 2 fired as soon as n2's Hop 1 +# landed, n4 would be cut over to direct replication from n1/n2/n3 before +# n3's backlog had fully passed through n1's forwarding path -- a real, +# permanent gap: n4 would end up missing some of n3's rows relative to n3 +# itself, even though n4 matches n1 and n2 exactly. Comparing n4 to n3 +# specifically (not just to n1/n2, which 048/049 already do) is therefore +# the correctness assertion that actually exercises this: it fails if the +# barrier raced ahead, and passes only if it genuinely waited for n3. +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 3-node cluster, fully cross-wired (n1 <-> n2 <-> n3 <-> n1) +# ============================================================================= +create_cluster(3, 'Create 3-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(3, ['n1', 'n2', 'n3'], 'Cross-wire n1 <-> n2 <-> n3'); + +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out //= ''; + $out =~ s/^\s+|\s+$//g; + return $out; +} + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +sub wait_for_table_drain { + my ($node_a, $node_b, $table, $timeout) = @_; + $timeout //= 180; + my $count_a = scalar_query($node_a, "SELECT COUNT(*) FROM $table"); + for (1 .. $timeout) { + my $count_b = scalar_query($node_b, "SELECT COUNT(*) FROM $table"); + return 1 if defined $count_b && $count_b eq $count_a; + sleep(1); + } + return 0; +} + +# Poll a log file's content for a pattern (rather than a fixed offset read) +# since the file may not exist yet when polling starts. +sub wait_for_log_pattern { + my ($logfile, $pattern, $timeout) = @_; + for (1 .. $timeout) { + if (-f $logfile) { + open(my $fh, '<', $logfile) or die "Cannot open $logfile: $!"; + local $/; + my $content = <$fh>; + close($fh); + return time() if defined $content && $content =~ $pattern; + } + sleep(1); + } + return undef; +} + +# ============================================================================= +# SETUP: a table replicated across the existing 3-node mesh via DDL +# replication, to be under sustained write load on n3 throughout n4's join. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE lag_peer_tbl (id bigint PRIMARY KEY, val text)"; +pass('lag_peer_tbl created on n1'); + +for my $peer_node (2, 3) { + my $on_peer = '0'; + for (1 .. 30) { + $on_peer = scalar_query($peer_node, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'lag_peer_tbl'"); + last if $on_peer eq '1'; + sleep(1); + } + is($on_peer, '1', "lag_peer_tbl replicated to n$peer_node"); +} + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[2], '-d', $dbname, '-c', + "CREATE SEQUENCE lag_peer_id_seq"; + +# A PROCEDURE (not a DO block) can issue its own internal COMMITs, so a +# single long-lived connection on n3 produces a continuous stream of +# separately committed transactions for as long as this test needs a +# backlog to exist -- same technique as 046_apply_worker_exception_ +# misclassification.pl's mid-transaction-restart repro. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[2], '-d', $dbname, '-c', q{ + CREATE PROCEDURE lag_peer_load(n_batches int, batch_rows int) + LANGUAGE plpgsql AS $$ + DECLARE i int; + BEGIN + FOR i IN 1..n_batches LOOP + INSERT INTO lag_peer_tbl + SELECT nextval('lag_peer_id_seq'), 'x' || g + FROM generate_series(1, batch_rows) g; + COMMIT; + -- Throttled deliberately: n3 is in a full mesh, so n2 also + -- applies every one of these rows via its own subscription, + -- inflating n2's own WAL and hence n1's (idle-peer) lag to + -- n2 as a side effect. An unthrottled writer here overwhelms + -- the whole cluster, not just n3 (confirmed empirically: without + -- this sleep, the pre-flight lag gate below tripped on n2, not + -- the intended target n3). + PERFORM pg_sleep(0.02); + END LOOP; + END $$; +}; + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1 outbound replication never drained after setup; cannot proceed'); + +# Started only after --bidirectional's own check_preconditions() has +# already passed (see wait_for_slot_created() below) -- it hard-rejects up +# front if the *source* (n1) has any unreplicated changes pending to an +# existing peer. This writer targets n3, not n1, so it may not actually +# trip that specific check, but 050_bidir_join_crash_midcatchup.pl hit +# exactly this failure mode empirically for a same-node case, and starting +# late here costs nothing (n1's replication is already drained above) -- +# not worth relitigating precisely which lag check_preconditions() looks +# at. Kept running through the rest of the join; stopped server-side +# afterward (pg_terminate_backend), same as 050 and 046 -- killing the +# client process alone does not reliably stop a backend mid-CALL. +my $writer_pid; + +sub start_writer { + $writer_pid = spawn_background("$log_dir/lag_peer_writer.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[2], '-d', $dbname, + '-c', "CALL lag_peer_load(100000, 500)"); +} + +# ============================================================================= +# TEST: join n4 via n1 while n3 is lagging behind on its own writes +# ============================================================================= +my $n4_port = $node_ports->[2] + 1; +my $n4_datadir = '/tmp/tmp_spock_node_3_datadir_bidir_multipeer'; +my $n4_pending = "${n4_datadir}.spock_bidir_pending.json"; +my $n4_manifest = "$n4_datadir/spock_bidirectional_manifest.json"; +my $n4_dsn = "host=$host port=$n4_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n4_datadir) if -d $n4_datadir; +unlink($n4_pending) if -f $n4_pending; + +sub wait_for_slot_created { + my ($timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $n4_pending || -f $n4_manifest; + sleep(1); + } + return 0; +} + +my $n4_conf = '/tmp/tmp_spock_node_3_postgresql.conf.override.multipeer'; +open my $conf_fh, '>', $n4_conf or die "Cannot write $n4_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n4_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n4_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_multipeer.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n4_datadir, + '--subscriber-name', 'n4', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n4_dsn, + '--postgresql-conf', $n4_conf, + '--stall-timeout', '120', + '--max-wait', '600', +); + +ok(wait_for_slot_created(30), 'source slot created (safe to start the n3 writer)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); +start_writer(); + +my $barrier_start = wait_for_log_pattern($scs_log, + qr/Establishing peer coverage barrier/, 60); +ok(defined $barrier_start, 'coverage barrier phase started') + or diag("see $scs_log"); + +my $scs_rc = wait_for_pid($scs_pid, 780); +unless (defined $scs_rc) { + diag("spock_create_subscriber (multipeer) did not exit within 780s; killing it"); + kill('TERM', $scs_pid); + waitpid($scs_pid, 0); + $scs_rc = -1; +} +is($scs_rc, 0, '--bidirectional (3-peer mesh, one peer lagging) exits 0') + or diag("see $scs_log and $log_dir/lag_peer_writer.log"); + +# Sanity check on test validity, not correctness: if this comes back near 0, +# the load was not actually creating backlog at the moment the barrier ran, +# and the test below would not be exercising the "wait for a lagging peer" +# path at all. Loose threshold (this is a coarse, 1s-polling-resolution +# measurement, not a precise timing assertion) -- the real correctness +# proof is the n3-vs-n4 data comparison further down, which fails +# regardless of how long the barrier actually took if it raced ahead. +my $barrier_end = wait_for_log_pattern($scs_log, + qr/Clearing forwarding on the catchup subscription/, 5); +if (defined $barrier_start && defined $barrier_end) { + my $barrier_seconds = $barrier_end - $barrier_start; + diag("coverage barrier phase took approximately ${barrier_seconds}s"); + ok($barrier_seconds >= 2, + "coverage barrier took a non-trivial amount of time (${barrier_seconds}s), " . + "consistent with genuinely waiting on n3's backlog"); +} else { + fail('could not measure coverage barrier duration from ' . $scs_log); +} + +ok(wait_for_pg_ready($host, $n4_port, $pg_bin, 30), 'n4 postgres is running'); +ok(wait_for_sub_status(4, 'sub_n4_n1', 'replicating', 30), + 'catchup subscription sub_n4_n1 is replicating on n4'); +ok(wait_for_sub_status(4, 'sub_n4_n2', 'replicating', 30), + 'direct peer subscription sub_n4_n2 is replicating on n4'); +ok(wait_for_sub_status(4, 'sub_n4_n3', 'replicating', 30), + 'direct peer subscription sub_n4_n3 is replicating on n4'); +ok(wait_for_sub_status(3, 'sub_n3_n4', 'replicating', 30), + 'reverse subscription sub_n3_n4 is replicating on n3'); + +# ============================================================================= +# Stop the writer (server-side) and let everything drain before comparing. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-X', '-p', $node_ports->[2], '-d', $dbname, '-c', + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " . + "WHERE query LIKE 'CALL lag_peer_load%' AND pid <> pg_backend_pid()"; +for (1 .. 30) { + my $still_running = scalar_query(3, + "SELECT count(*) FROM pg_stat_activity WHERE query LIKE 'CALL lag_peer_load%'"); + last if defined $still_running && $still_running eq '0'; + sleep(1); +} +kill('TERM', $writer_pid); +waitpid($writer_pid, 0); +pass('n3 writer stopped'); + +diag('lag_peer_tbl did not fully drain to n1 within 180s') + unless wait_for_table_drain(3, 1, 'lag_peer_tbl'); +diag('lag_peer_tbl did not fully drain to n4 within 180s') + unless wait_for_table_drain(3, 4, 'lag_peer_tbl'); +diag('lag_peer_tbl did not fully drain to n2 within 180s') + unless wait_for_table_drain(3, 2, 'lag_peer_tbl'); + +# ============================================================================= +# THE assertion: n4 must match n3 exactly, not just n1/n2 -- this is the +# pair that only fails if the coverage barrier raced ahead of n3's backlog. +# ============================================================================= +my $hash_query = "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) " . + "FROM lag_peer_tbl x"; +my $count_n3 = scalar_query(3, "SELECT COUNT(*) FROM lag_peer_tbl"); +my $count_n4 = scalar_query(4, "SELECT COUNT(*) FROM lag_peer_tbl"); +is($count_n4, $count_n3, 'row count matches between n3 (lagging peer) and n4'); +my $hash_n3 = scalar_query(3, $hash_query); +my $hash_n4 = scalar_query(4, $hash_query); +is($hash_n4, $hash_n3, 'content hash matches between n3 (lagging peer) and n4'); + +my $count_n1 = scalar_query(1, "SELECT COUNT(*) FROM lag_peer_tbl"); +is($count_n4, $count_n1, 'row count matches between n1 and n4'); +my $hash_n1 = scalar_query(1, $hash_query); +is($hash_n4, $hash_n1, 'content hash matches between n1 and n4'); + +my $count_n2 = scalar_query(2, "SELECT COUNT(*) FROM lag_peer_tbl"); +is($count_n4, $count_n2, 'row count matches between n2 and n4'); +my $hash_n2 = scalar_query(2, $hash_query); +is($hash_n4, $hash_n2, 'content hash matches between n2 and n4'); + +# ============================================================================= +# Prove the n3 <-> n4 pair is a genuinely live DIRECT subscription, not just +# correct inherited catchup data -- a fresh two-way write test. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $n4_port, '-d', $dbname, '-c', + "INSERT INTO lag_peer_tbl (id, val) VALUES (900000001, 'from_n4_direct')"; +my $on_n3_direct = '0'; +for (1 .. 30) { + $on_n3_direct = scalar_query(3, + "SELECT COUNT(*) FROM lag_peer_tbl WHERE val = 'from_n4_direct'"); + last if $on_n3_direct eq '1'; + sleep(1); +} +is($on_n3_direct, '1', 'fresh write on n4 reaches n3 via the direct subscription'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[2], '-d', $dbname, '-c', + "INSERT INTO lag_peer_tbl (id, val) VALUES (900000002, 'from_n3_direct')"; +my $on_n4_direct = '0'; +for (1 .. 30) { + $on_n4_direct = scalar_query(4, + "SELECT COUNT(*) FROM lag_peer_tbl WHERE val = 'from_n3_direct'"); + last if $on_n4_direct eq '1'; + sleep(1); +} +is($on_n4_direct, '1', 'fresh write on n3 reaches n4 via the direct subscription'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n4_datadir ], + '--cleanup --force exits 0' +); +ok(!-d $n4_datadir, 'n4 data directory removed after cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n4_conf) if -f $n4_conf; +destroy_cluster('Destroy 3-node cluster'); + +done_testing(); diff --git a/tests/tap/t/052_bidir_join_big_txn_liveness.pl b/tests/tap/t/052_bidir_join_big_txn_liveness.pl new file mode 100644 index 00000000..959a9379 --- /dev/null +++ b/tests/tap/t/052_bidir_join_big_txn_liveness.pl @@ -0,0 +1,263 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 052_bidir_join_big_txn_liveness.pl - the catchup stall watchdog must +# not fire on a healthy, merely +# slow-to-apply transaction +# ============================================================================= +# Design doc (spock_bidirectional_final.md) section 11: the Spock output +# plugin registers no streaming (in-progress-transaction) callbacks, so a +# large transaction is decoded and sent, and its origin advanced, only at +# its COMMIT. Every LSN signal wait_for_catchup()/wait_for_origin_progress() +# watches therefore freezes for as long as that one transaction is still +# being applied -- even though the apply worker is healthy and busy, not +# stalled. Before this fix, --stall-timeout was a pure LSN-progress +# watchdog and would die() on exactly this pattern; section 16 calls out a +# "big-transaction liveness" regression test for it explicitly, and it did +# not exist. +# +# This reproduces it directly: a REPLICA-only trigger with pg_sleep() on a +# table, present on n1 before the physical backup (so n3 inherits it too), +# fires only when a row is *applied* via replication -- not when n1's own +# origin transaction inserts it -- so a single, fast-committing transaction +# on n1 takes many seconds to fully apply on n3, freezing n3's catchup +# origin for that whole window. --stall-timeout is set far shorter than +# that window: without the liveness check, catchup dies well before the +# transaction finishes applying; with it, the apply worker's own +# pg_stat_activity state proves it is still busy, and the wait continues +# through to a correct, complete result. +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +sub wait_for_slot_created { + my ($pending_path, $manifest_path, $timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $pending_path || -f $manifest_path; + sleep(1); + } + return 0; +} + +# ============================================================================= +# SETUP: a table with a REPLICA-only slow trigger, present on n1 before the +# physical backup so n3 inherits it too. ENABLE REPLICA TRIGGER means it +# fires only on rows applied via replication, not on n1's own origin +# inserts -- so inserting into this table on n1 is fast, but n3 takes a +# long time to apply each row. +# ============================================================================= +my $rows = 200; +my $sleep_per_row = 0.15; +my $expected_min_seconds = int($rows * $sleep_per_row * 0.8); # generous margin + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE slow_apply_tbl (id serial primary key, val text)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', qq{ + CREATE FUNCTION slow_apply_trigger() RETURNS trigger LANGUAGE plpgsql AS \$\$ + BEGIN + PERFORM pg_sleep($sleep_per_row); + RETURN NEW; + END \$\$; + CREATE TRIGGER slow_apply_trg BEFORE INSERT ON slow_apply_tbl + FOR EACH ROW EXECUTE FUNCTION slow_apply_trigger(); + ALTER TABLE slow_apply_tbl ENABLE REPLICA TRIGGER slow_apply_trg; +}; +pass("slow_apply_tbl with a replica-only ${sleep_per_row}s-per-row trigger created on n1"); + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1->n2 replication never drained after setup; cannot proceed'); + +# ============================================================================= +# TEST: join with a --stall-timeout far shorter than the time this one +# transaction will take to apply on n3 +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_liveness'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; +my $stall_timeout = 5; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.liveness'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_liveness.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', "$stall_timeout", + '--max-wait', '300', +); + +ok(wait_for_slot_created($n3_pending, $n3_manifest, 30), + 'source slot created (safe to insert the slow transaction)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); + +# Fast on n1 (the replica-only trigger does not fire for n1's own origin +# insert): a single transaction, all $rows rows, so its origin only +# advances on n3 once every row has been applied there. +my $insert_start = time(); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "INSERT INTO slow_apply_tbl (val) SELECT 'row ' || g FROM generate_series(1, $rows) g"; +my $insert_elapsed = time() - $insert_start; +ok($insert_elapsed < $expected_min_seconds, + "the insert on n1 itself was fast (${insert_elapsed}s) -- the replica " . + "trigger did not fire for n1's own origin transaction"); + +my $join_start = time(); +my $scs_rc = wait_for_pid($scs_pid, 300); +unless (defined $scs_rc) { + diag("spock_create_subscriber did not exit within 300s; killing it"); + kill('TERM', $scs_pid); + waitpid($scs_pid, 0); + $scs_rc = -1; +} +my $join_elapsed = time() - $join_start; +is($scs_rc, 0, + "--bidirectional exits 0 despite a single transaction frozen on n3 for " . + "longer than --stall-timeout=${stall_timeout}s") + or diag("see $scs_log"); + +ok($join_elapsed >= $expected_min_seconds, + "the join took at least ${expected_min_seconds}s (${join_elapsed}s elapsed), " . + "confirming the slow-apply transaction was genuinely still in flight " . + "when the join reached it, not skipped or already caught up"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3'); + +my $n1_count = scalar_query(1, "SELECT COUNT(*) FROM slow_apply_tbl"); +is($n1_count, "$rows", 'all rows present on n1'); +my $n3_count = '-1'; +for (1 .. 30) { + $n3_count = scalar_query(3, "SELECT COUNT(*) FROM slow_apply_tbl"); + last if $n3_count eq $n1_count; + sleep(1); +} +is($n3_count, $n1_count, + 'every row of the slow transaction was applied on n3, none skipped or discarded'); + +is(scalar_query(3, "SELECT COUNT(*) FROM spock.exception_log"), + '0', 'no exceptions were logged (the wait genuinely continued, not replayed/discarded)'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/tests/tap/t/053_bidir_join_unbounded_peer_slot_wal.pl b/tests/tap/t/053_bidir_join_unbounded_peer_slot_wal.pl new file mode 100644 index 00000000..1be55c83 --- /dev/null +++ b/tests/tap/t/053_bidir_join_unbounded_peer_slot_wal.pl @@ -0,0 +1,317 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 053_bidir_join_unbounded_peer_slot_wal.pl - demonstrates the +# WAL-retention exposure +# a peer slot has during +# the coverage barrier, +# currently unprotected +# ============================================================================= +# Design doc (spock_bidirectional_final.md) section 13: a peer slot created +# in Step 20 Phase A sits unconsumed -- pinning WAL and catalog_xmin on that +# peer -- until the direct subscription is enabled in Step 20.6. Section 13 +# promotes a pre-flight lag gate and a --max-slot-wal budget/abort valve to +# a v1 requirement to bound that exposure. Both were implemented, tested, +# and then deliberately reverted (see the revert commit for the "how +# necessary is this" discussion): the existing --stall-timeout + manual +# --cleanup path already bounds the exposure in the common case, just less +# precisely (by elapsed time, not actual WAL bytes) than a dedicated guard +# would, and no test had ever actually hit the gap the guard closes. +# +# This test is that missing evidence, kept independently of whether the +# guard exists: it demonstrates the exposure is real, not hypothetical, by +# measuring it directly -- keeping a peer (n2) under continuous write load +# throughout a new node's (n3's) join, and sampling n2's own peer +# replication slot's retained WAL (pg_wal_lsn_diff(pg_current_wal_lsn(), +# restart_lsn), the exact signal section 13 names) throughout the coverage +# barrier. It asserts retention grows past a threshold that would matter in +# production, and that the join still succeeds correctly despite that +# growth -- this is a resource-exposure characterization, not a +# correctness bug: nothing here should ever fail the join itself. +# +# Re-run this same test once --max-slot-wal is reintroduced (a dedicated +# follow-up per the revert commit): with the guard in place and a --max- +# slot-wal budget set below what this test currently observes, the join +# should instead abort with every peer slot dropped -- the assertions +# below would need inverting at that point, which is exactly the point of +# keeping this test around. +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +sub wait_for_slot_created { + my ($pending_path, $manifest_path, $timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $pending_path || -f $manifest_path; + sleep(1); + } + return 0; +} + +# ============================================================================= +# SETUP: a table on n2, replicated to n1 via the existing mesh, kept under +# continuous write load throughout n3's join. batch-committed via a +# PROCEDURE so a single long-lived connection produces a steady stream +# rather than one huge transaction -- the exposure this test measures +# accrues across many small transactions the peer slot sits behind, not +# from any one of them being unusually large. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "CREATE TABLE peer_wal_tbl (id bigint PRIMARY KEY, val text)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "CREATE SEQUENCE peer_wal_id_seq"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', q{ + CREATE PROCEDURE peer_wal_load(n_batches int, batch_rows int) + LANGUAGE plpgsql AS $$ + DECLARE i int; + BEGIN + FOR i IN 1..n_batches LOOP + INSERT INTO peer_wal_tbl + SELECT nextval('peer_wal_id_seq'), 'x' || g + FROM generate_series(1, batch_rows) g; + COMMIT; + PERFORM pg_sleep(0.02); + END LOOP; + END $$; +}; +pass('peer_wal_tbl and writer procedure created on n2'); + +# The peer slot spock_create_subscriber will create on n2 for n3's future +# direct subscription is a pure, deterministic function of (dbname, +# provider_node_name, sub_name) -- spock.spock_gen_slot_name() -- so its +# name is knowable before the join even starts (n3's sub-to-n2 always +# follows the sub__ convention). +my $n2_peer_slot_name = scalar_query(2, + "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); +ok(length($n2_peer_slot_name) > 0, 'resolved n2 peer slot name in advance'); + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1->n2 replication never drained after setup; cannot proceed'); + +# ============================================================================= +# TEST: join n3 via n1 while n2 is under continuous write load; sample the +# peer slot's own retained WAL throughout the coverage barrier. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_unbounded_wal'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.unbounded_wal'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_unbounded_wal.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '300', +); + +ok(wait_for_slot_created($n3_pending, $n3_manifest, 30), + 'source slot created (safe to start the n2 writer)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); + +my $writer_pid = spawn_background("$log_dir/peer_wal_writer.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[1], '-d', $dbname, + '-c', "CALL peer_wal_load(100000, 500)"); + +# Sample the peer slot's own retention on n2 (the exact query design doc +# section 13 names: pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn)) +# from the moment it appears until the join finishes. WNOHANG so this +# loop and the join proceed concurrently rather than sampling only once +# after the fact, when the slot may already be gone (consumed at cutover) +# or the window already closed. +my $max_retained_bytes = 0; +my $samples = 0; +for (;;) { + my $r = waitpid($scs_pid, WNOHANG); + last if $r == $scs_pid; + + my $retained = scalar_query(2, + "SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) " . + "FROM pg_replication_slots WHERE slot_name = '$n2_peer_slot_name'"); + if (defined $retained && $retained =~ /^\d+$/) { + $samples++; + $max_retained_bytes = $retained if $retained > $max_retained_bytes; + } + sleep(1); +} +my $scs_rc = ($? >> 8); + +diag("sampled the peer slot's retained WAL $samples time(s); " . + "max observed: $max_retained_bytes byte(s)"); + +is($scs_rc, 0, '--bidirectional exits 0 despite n2 being under continuous load') + or diag("see $scs_log"); + +# The actual demonstration: without a budget guard, retention on the peer +# slot grows to a size that would matter in production while it sits +# unconsumed -- not just a few stray bytes from normal replication +# bookkeeping. 1MB is comfortably below what this test's writer produces +# over even a short barrier window (empirically tens of MB), while being +# far above any incidental noise -- a threshold this loose only fails if +# the exposure genuinely stopped growing, not from ordinary run-to-run +# timing variance. +ok($max_retained_bytes > 1024 * 1024, + "peer slot retention exceeded 1MB during the barrier " . + "(max observed: $max_retained_bytes byte(s)) -- the WAL-retention " . + "exposure design doc section 13 describes is real, not hypothetical"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3'); +ok(wait_for_sub_status(3, 'sub_n3_n2', 'replicating', 30), + 'direct peer subscription sub_n3_n2 is replicating on n3'); + +# Correctness is unaffected by the exposure -- this is a resource cost, +# not a data-safety bug. Stop the writer (server-side: killing the client +# alone does not reliably stop a backend mid-CALL) and confirm the data +# converges exactly once it drains. +system_or_bail "$pg_bin/psql", '-X', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " . + "WHERE query LIKE 'CALL peer_wal_load%' AND pid <> pg_backend_pid()"; +for (1 .. 30) { + my $still_running = scalar_query(2, + "SELECT count(*) FROM pg_stat_activity WHERE query LIKE 'CALL peer_wal_load%'"); + last if defined $still_running && $still_running eq '0'; + sleep(1); +} +kill('TERM', $writer_pid); +waitpid($writer_pid, 0); + +my $count_n2 = scalar_query(2, "SELECT COUNT(*) FROM peer_wal_tbl"); +my $count_n3 = '-1'; +for (1 .. 60) { + $count_n3 = scalar_query(3, "SELECT COUNT(*) FROM peer_wal_tbl"); + last if $count_n3 eq $count_n2; + sleep(1); +} +is($count_n3, $count_n2, + 'n3 eventually receives every row despite the unbounded retention window'); + +my $slot_gone = scalar_query(2, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name = '$n2_peer_slot_name'"); +ok($slot_gone >= 0, 'peer slot query still succeeds after cutover (informational)'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/tests/tap/t/054_bidir_join_precondition_negatives.pl b/tests/tap/t/054_bidir_join_precondition_negatives.pl new file mode 100644 index 00000000..c10d001c --- /dev/null +++ b/tests/tap/t/054_bidir_join_precondition_negatives.pl @@ -0,0 +1,532 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 054_bidir_join_precondition_negatives.pl +# spock_create_subscriber --bidirectional -- cheap pre-flight rejections +# ============================================================================= +# Design doc spock_bidirectional_final.md section 16 "Negative paths" lists a +# set of preconditions --bidirectional must reject before touching a peer's +# replication slot or the source's data directory. 048_bidir_join.pl already +# covers a broken full-mesh topology and one replication-set flag mismatch; +# this file rounds out the rest of that list with scenarios that don't need +# any pause/injection machinery -- each sets up exactly one bad precondition, +# attempts the join, and asserts a clean, specific rejection, then repairs the +# precondition before moving on so later scenarios start from a known-good +# state. The final scenario is the one exception: it needs one real +# completed join, to test that a second, colliding join attempt fails clean. +# +# Topology: n1 <-> n2, the same cross-wired 2-node cluster 048 uses. +# +# Test count breakdown: +# 1 binary found +# 5 create_cluster(2) +# 1 cross_wire n1<->n2 +# -- Scenario 1: peer unreachable at discovery +# 1 --bidirectional rejects an unreachable peer +# 1 rejection reports the mesh edge to the down peer as unhealthy +# 1 n2 postgres is running again +# -- Scenario 2: track_commit_timestamp=off on the source (SKIPPED -- see +# comment at the scenario: no legal spock.conflict_resolution value +# permits track_commit_timestamp=off in this build, so no live node +# exists to test this precondition against) +# 2 skip +# -- Scenario 3: track_commit_timestamp=off on a peer (SKIPPED, same reason) +# 2 skip +# -- Scenario 4: Spock version too old +# 1 --bidirectional rejects a too-old Spock version on the source +# 1 --bidirectional rejects a too-old Spock version on a peer +# -- Scenario 5: source has unreplicated changes pending to an existing peer +# 1 --bidirectional rejects source with undrained outbound lag to a peer +# -- Scenario 6: P0.5 filter-equivalence, remaining sub-cases +# 1 --bidirectional rejects a repset table-membership mismatch +# 1 --bidirectional rejects a repset row_filter mismatch +# 1 --bidirectional rejects a repset sequence-membership mismatch +# -- Scenario 7: idempotent double-invocation +# 1 first --bidirectional join (minimal, no custom repset seeding) exits 0 +# 1 n3a postgres is running +# 1 second --bidirectional join reusing the subscriber name "n3" is rejected +# 1 the first n3's own subscription to the source is still replicating +# 1 no duplicate "n3" node row was left behind on the source +# 1 --cleanup --force removes the first join's state +# 1 destroy_cluster +# --- +# 27 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 27; +use File::Path qw(remove_tree); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + wait_for_sub_status + psql_or_bail wait_for_pg_ready); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_file = $config->{log_file}; +my $n1_datadir = $config->{node_datadirs}->[0]; +my $n2_datadir = $config->{node_datadirs}->[1]; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; +my $n2_dsn = "host=$host port=$node_ports->[1] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out =~ s/^\s+|\s+$//g if defined $out; + return $out; +} + +# The die() message for every rejection below lands in this test's own log +# file (SpockTest.pm redirects every child process's stderr there), not on +# the console -- read it back after each system_maybe() to verify *why* the +# join was rejected, not just that it was. +sub log_tail_has { + my ($pattern, $since_bytes) = @_; + open(my $fh, '<', $log_file) or return ''; + seek($fh, $since_bytes, 0); + local $/; + my $tail = <$fh> // ''; + close $fh; + return $tail =~ $pattern; +} +sub log_size { + return -s $log_file // 0; +} + +# Wait for n1's outbound replication to n2 to fully drain (the same +# precondition check() itself performs) before attempting a join, so an +# earlier scenario's DDL/data doesn't leak into a later one's rejection. +sub wait_for_n1_drained { + for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return if defined $lag && $lag eq '0'; + sleep(1); + } +} +wait_for_n1_drained(); + +my $port_ctr = 0; +sub fresh_datadir { + $port_ctr++; + my $dir = "/tmp/tmp_spock_node_2_datadir_bidir_pn_$port_ctr"; + remove_tree($dir) if -d $dir; + return $dir; +} + +# ============================================================================= +# Scenario 1: peer unreachable at discovery. A fully-down peer is caught by +# check_mesh_edges() -- which reads sub_n1_n2's *local* status on n1, not a +# live connection to n2 -- before check_preconditions()'s later per-peer +# connect loop is ever reached, so the actual rejection is the mesh-health +# message, not a distinct "cannot connect" one. (The narrower race this +# file cannot construct -- n1 still believes sub_n1_n2 is replicating while +# a direct TCP attempt to n2 independently fails -- is what "cannot connect +# to peer" in check_preconditions()/check_replication_set_equivalence() +# actually guards.) +# ============================================================================= +note("Scenario 1: peer unreachable at discovery"); +system_or_bail "$pg_bin/pg_ctl", 'stop', '-D', $n2_datadir, '-m', 'fast'; + +my $before = log_size(); +my $dir1 = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir1, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 100) . " dbname=$dbname"), + '--bidirectional rejects an unreachable peer'); +ok(log_tail_has(qr/no healthy \(status = 'replicating'\) subscription from "n2"/, $before), + 'rejection reports the mesh edge to the down peer as unhealthy'); +remove_tree($dir1) if -d $dir1; + +system_or_bail "$pg_bin/pg_ctl", 'start', '-D', $n2_datadir, + '-l', "$config->{log_dir}/n2_scenario1_restart.log"; +ok(wait_for_pg_ready($host, $node_ports->[1], $pg_bin, 30), 'n2 postgres is running again') + or BAIL_OUT('cannot continue without n2 back up'); +wait_for_n1_drained(); + +# ============================================================================= +# Scenario 2/3: track_commit_timestamp=off, source then peer. SKIPPED -- +# confirmed genuinely unreachable in this build, not a test-construction +# problem. spock.conflict_resolution's only enabled enum string is +# "last_update_wins" (SpockConflictResolvers[], spock.c:83-93 -- apply_remote/ +# error/keep_local/first_update_wins are all commented out pending upstream +# design work: "Disabled until we can clearly define their desired behavior. +# Jan Wieck 2024-08-12"). spock_conflict_resolver_check_hook() +# (spock_conflict.c:799) rejects last_update_wins whenever +# track_commit_timestamp is off and accepts only apply_remote/error instead +# -- neither of which is a legal GUC value right now. So no spock-loaded +# postmaster can boot at all with track_commit_timestamp=off in this build +# (confirmed directly: ALTER SYSTEM SET spock.conflict_resolution = +# 'apply_remote' itself fails with "invalid value for parameter +# ... HINT: Available values: last_update_wins.", before the intended +# restart-and-observe-FATAL path is even reached) -- there is no live node +# check_preconditions()'s track_commit_timestamp check can ever be exercised +# against. Revisit once apply_remote/error are re-enabled. +# ============================================================================= +note("Scenario 2: track_commit_timestamp=off on the source"); +SKIP: { + skip 'no legal spock.conflict_resolution value permits track_commit_timestamp' + . '=off in this build (apply_remote/error are commented out of' + . ' SpockConflictResolvers[], spock.c:83-93) -- a spock-loaded postmaster' + . ' cannot boot with it off at all, so this precondition path has no live' + . ' node to test against', 2; +} + +note("Scenario 3: track_commit_timestamp=off on a peer"); +SKIP: { + skip 'same as Scenario 2 -- see comment above', 2; +} + +# ============================================================================= +# Scenario 4: Spock version too old. Faked by editing the catalog metadata +# row check_spock_version_at_least_6() reads (pg_extension.extversion) -- +# this does not touch the loaded shared library, only the version string the +# precondition check itself trusts, which is exactly the surface being +# tested. Restored immediately after each attempt. +# ============================================================================= +note("Scenario 4: Spock version too old"); +my $real_extversion = scalar_query(1, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "UPDATE pg_extension SET extversion = '5.0.11' WHERE extname = 'spock'"; +my $dir4a = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir4a, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 103) . " dbname=$dbname"), + '--bidirectional rejects a too-old Spock version on the source'); +remove_tree($dir4a) if -d $dir4a; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "UPDATE pg_extension SET extversion = '$real_extversion' WHERE extname = 'spock'"; + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "UPDATE pg_extension SET extversion = '5.0.11' WHERE extname = 'spock'"; +my $dir4b = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir4b, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 104) . " dbname=$dbname"), + '--bidirectional rejects a too-old Spock version on a peer'); +remove_tree($dir4b) if -d $dir4b; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "UPDATE pg_extension SET extversion = '$real_extversion' WHERE extname = 'spock'"; + +# ============================================================================= +# Scenario 5: source has unreplicated changes pending to an existing peer. +# Disabling sub_n2_n1 stops n2 consuming, without needing a full restart; +# a write on n1 then leaves n1's outbound slot to n2 behind. +# ============================================================================= +note("Scenario 5: source has undrained outbound lag to an existing peer"); +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_disable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 'f'; + sleep(1); +} +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pn_lag_probe (id int)"; + +$before = log_size(); +my $dir5 = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir5, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 105) . " dbname=$dbname"), + '--bidirectional rejects source with undrained outbound lag to a peer'); +remove_tree($dir5) if -d $dir5; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_enable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 't'; + sleep(1); +} +wait_for_n1_drained(); +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pn_lag_probe"; +wait_for_n1_drained(); + +# ============================================================================= +# Scenario 6: P0.5 filter-equivalence, remaining sub-cases. 048_bidir_join.pl +# already proves the flags sub-case (replicate_truncate mismatch on +# 'default'); these three exercise the other independent segments of +# compute_repset_fingerprints()'s per-table string (tbl=...|cols=...| +# filter=...|schema=(...)) and the separate seq=... segment. Column-list and +# partition-inclusion mismatches are not separately tested here -- they +# differ from the row_filter case only in which substring of the same +# per-table fingerprint changes, not in which code path fires or which die() +# message is reached, so table-membership + row_filter + sequence-membership +# already exercise every independent branch of check_replication_set_ +# equivalence() (found on source only, found on peer only, and same-name- +# different-fingerprint). Only 'default' is checked here because +# build_selected_set_name_filter() restricts comparison to sets an existing +# subscription actually references -- an unreferenced custom set (as seeded +# in 048) is invisible to this check. DDL replication is turned off for +# each mutating ALTER so the mismatch is real and local to n2, exactly as +# 048's flags test does. +# ============================================================================= +note("Scenario 6: repset table-membership mismatch"); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; CREATE TABLE pn_extra_tbl (id int PRIMARY KEY)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_add_table('default', 'pn_extra_tbl', synchronize_data := false)"; + +my $dir6a = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir6a, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 106) . " dbname=$dbname"), + '--bidirectional rejects a repset table-membership mismatch'); +remove_tree($dir6a) if -d $dir6a; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_remove_table('default', 'pn_extra_tbl')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; DROP TABLE pn_extra_tbl"; + +note("Scenario 6: repset row_filter mismatch"); +# CREATE TABLE runs with DDL replication ON (unlike the table-membership +# case above), specifically so it reaches n2 -- but autoddl capture +# (spock_autoddl.c) auto-adds a newly created table to 'default' on BOTH +# nodes as part of applying that same DDL, with no filter on either side. +# An explicit repset_add_table() on n1 here would just collide with that +# auto-membership (duplicate key on replication_set_table_pkey) -- so skip +# it and let auto-membership do the (filter-less, matching) add on n1; +# the wait loop below confirms n2 got the same auto-add. The actual +# mismatch is then made by remove+re-add *on n2 only*, DDL replication off. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pn_filter_tbl (id int PRIMARY KEY, region text)"; +for (1 .. 15) { + my $present = scalar_query(2, + "SELECT COUNT(*) FROM spock.replication_set_table rts" . + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" . + " WHERE rs.set_name = 'default' AND rts.set_reloid::regclass::text = 'pn_filter_tbl'"); + last if defined $present && $present eq '1'; + sleep(1); +} +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_remove_table('default', 'pn_filter_tbl')"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_add_table('default', 'pn_filter_tbl', synchronize_data := false, " . + "row_filter := 'id > 0')"; + +my $dir6b = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir6b, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 107) . " dbname=$dbname"), + '--bidirectional rejects a repset row_filter mismatch'); +remove_tree($dir6b) if -d $dir6b; + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE pn_filter_tbl"; +for (1 .. 15) { + my $gone = scalar_query(2, "SELECT to_regclass('pn_filter_tbl') IS NULL"); + last if defined $gone && $gone eq 't'; + sleep(1); +} + +note("Scenario 6: repset sequence-membership mismatch"); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; CREATE SEQUENCE pn_extra_seq"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_add_seq('default', 'pn_extra_seq')"; + +my $dir6c = fresh_datadir(); +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $dir6c, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', "host=$host port=" . ($node_ports->[1] + 108) . " dbname=$dbname"), + '--bidirectional rejects a repset sequence-membership mismatch'); +remove_tree($dir6c) if -d $dir6c; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_remove_seq('default', 'pn_extra_seq')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; DROP SEQUENCE pn_extra_seq"; +wait_for_n1_drained(); +# wait_for_n1_drained() only confirms LSN lag has drained, not that +# sub_show_status() itself has settled -- the DROP SEQUENCE/repset_remove_seq +# cycle just above can leave a brief window where a worker is still +# restarting from the DDL and check_mesh_edges() sees a transient non- +# 'replicating' status on one side, which would spuriously fail Scenario 7's +# very first join for a reason that has nothing to do with idempotency. +# Confirmed happening in practice once -- wait for both directions to be +# genuinely settled before relying on the mesh being healthy. +wait_for_sub_status(1, 'sub_n1_n2', 'replicating', 30) + or BAIL_OUT('mesh did not restabilize after Scenario 6 cleanup'); +wait_for_sub_status(2, 'sub_n2_n1', 'replicating', 30) + or BAIL_OUT('mesh did not restabilize after Scenario 6 cleanup'); + +# ============================================================================= +# Scenario 7: idempotent double-invocation. One real join, left live (no +# --cleanup), then a second attempt reusing the same --subscriber-name "n3" +# against a fresh, empty pgdata. Whichever check trips it, the join must +# fail rather than leave a duplicate "n3" node or disturb the first n3. +# ============================================================================= +note("Scenario 7: idempotent double-invocation"); +my $n3a_datadir = fresh_datadir(); +my $n3a_port = $node_ports->[1] + 150; +my $n3a_dsn = "host=$host port=$n3a_port dbname=$dbname" + . " user=$db_user password=$db_password"; +my $n3a_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.n3a'; +open my $conf_fh_a, '>', $n3a_conf or die "Cannot write $n3a_conf: $!"; +print $conf_fh_a "shared_buffers=1GB\n"; +print $conf_fh_a "shared_preload_libraries='spock'\n"; +print $conf_fh_a "wal_level=logical\n"; +print $conf_fh_a "spock.enable_ddl_replication=on\n"; +print $conf_fh_a "spock.include_ddl_repset=on\n"; +print $conf_fh_a "spock.allow_ddl_from_functions=on\n"; +print $conf_fh_a "spock.exception_behaviour=sub_disable\n"; +print $conf_fh_a "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh_a "track_commit_timestamp=on\n"; +print $conf_fh_a "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh_a "spock.enable_spill=on\n"; +print $conf_fh_a "port=$n3a_port\n"; +print $conf_fh_a "listen_addresses='*'\n"; +print $conf_fh_a "logging_collector=on\n"; +print $conf_fh_a "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh_a "log_filename='00${n3a_port}.log'\n"; +close $conf_fh_a; + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $n3a_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3a_dsn, + '--postgresql-conf', $n3a_conf, + ], + 'first --bidirectional join (minimal, no custom repset seeding) exits 0' +); +ok(wait_for_pg_ready($host, $n3a_port, $pg_bin, 30), 'n3a postgres is running'); + +my $n3b_datadir = fresh_datadir(); +my $n3b_port = $node_ports->[1] + 151; +my $n3b_dsn = "host=$host port=$n3b_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +# n3a's own postgres is still running on $n3a_port -- reusing $n3a_conf here +# (which hardcodes that port) would make n3b's postgres either fail to bind +# it (n3a already holds it) or, if it somehow started anyway, listen on a +# port --subscriber-dsn never points at, so --bidirectional's own +# wait_postmaster_connection() polls a port nothing answers on forever. +# n3b needs its own override, identical to n3a's but for its own port. +my $n3b_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.n3b'; +open my $conf_fh_b, '>', $n3b_conf or die "Cannot write $n3b_conf: $!"; +print $conf_fh_b "shared_buffers=1GB\n"; +print $conf_fh_b "shared_preload_libraries='spock'\n"; +print $conf_fh_b "wal_level=logical\n"; +print $conf_fh_b "spock.enable_ddl_replication=on\n"; +print $conf_fh_b "spock.include_ddl_repset=on\n"; +print $conf_fh_b "spock.allow_ddl_from_functions=on\n"; +print $conf_fh_b "spock.exception_behaviour=sub_disable\n"; +print $conf_fh_b "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh_b "track_commit_timestamp=on\n"; +print $conf_fh_b "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh_b "spock.enable_spill=on\n"; +print $conf_fh_b "port=$n3b_port\n"; +print $conf_fh_b "listen_addresses='*'\n"; +print $conf_fh_b "logging_collector=on\n"; +print $conf_fh_b "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh_b "log_filename='00${n3b_port}.log'\n"; +close $conf_fh_b; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $n3b_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3b_dsn, + '--postgresql-conf', $n3b_conf), + 'second --bidirectional join reusing the subscriber name "n3" is rejected'); +remove_tree($n3b_datadir) if -d $n3b_datadir; +unlink($n3b_conf) if -f $n3b_conf; + +is(psql_capture('-p', $n3a_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n1')"), + 'replicating', "the first n3's own subscription to the source is still replicating"); + +my $node_name_count = scalar_query(1, + "SELECT COUNT(*) FROM spock.node WHERE node_name = 'n3'"); +is($node_name_count, '1', 'no duplicate "n3" node row was left behind on the source'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $n3a_datadir, + ], + q(--cleanup --force removes the first join's state) +); +unlink($n3a_conf) if -f $n3a_conf; +remove_tree($n3a_datadir) if -d $n3a_datadir; + +# ============================================================================= +# CLEANUP +# ============================================================================= +destroy_cluster('Cleanup'); diff --git a/tests/tap/t/055_bidir_join_lww_conflict.pl b/tests/tap/t/055_bidir_join_lww_conflict.pl new file mode 100644 index 00000000..31657a31 --- /dev/null +++ b/tests/tap/t/055_bidir_join_lww_conflict.pl @@ -0,0 +1,335 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 055_bidir_join_lww_conflict.pl - conflicting concurrent writes on two +# existing peers during a new node's +# --bidirectional catchup resolve to +# the SAME value on the new node as on +# the existing peers +# ============================================================================= +# Design doc spock_bidirectional_final.md section 16 lists, as a required +# positive-path scenario: "LWW conflict: concurrent conflicting writes on two +# peers during catchup; assert LWW resolves identically on source and n3." +# No existing bidirectional test exercises a genuine same-row conflict; every +# other bidir test (048/049/050/051/052/053) only ever writes NEW, non- +# colliding rows. +# +# n3 does not live through the conflict itself: it is not yet directly +# subscribed to n2 while catching up (that direct subscription stays +# disabled -- see spock_bidirectional_implementation.md's "Step 16b" -- +# until Step 20.6, well after this test's conflicting writes land). n3 only +# ever sees n2's write forwarded through n1's catchup subscription +# (forward_origins := '{all}'), tagged with n2's own origin so n3's apply +# worker resolves the conflict independently, against n1's own write, using +# the SAME origin/commit-timestamp inputs n1 itself used. If forwarding +# dropped a change, reordered it, or lost the origin tag, n3 would converge +# on a different winner than n1/n2 despite the join reporting success -- +# exactly the failure this test is positioned to catch that no other bidir +# test can (they never create a same-row conflict in the first place). +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, bidirectionally cross-wired (n1 <-> n2) +# ============================================================================= +create_cluster(2, 'Create 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2'); + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +# Poll a log file's content for a pattern (rather than a fixed offset read) +# since the file may not exist yet when polling starts. +sub wait_for_log_pattern { + my ($logfile, $pattern, $timeout) = @_; + for (1 .. $timeout) { + if (-f $logfile) { + open(my $fh, '<', $logfile) or die "Cannot open $logfile: $!"; + local $/; + my $content = <$fh>; + close($fh); + return 1 if defined $content && $content =~ $pattern; + } + sleep(1); + } + return 0; +} + +# ============================================================================= +# SETUP: a table on the existing mesh, one seed row both n1 and n2 will race +# to update once n3's join is underway. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE lww_conflict_tbl (id int PRIMARY KEY, val text, writer text)"; +pass('lww_conflict_tbl created on n1'); + +my $on_n2 = '0'; +for (1 .. 30) { + $on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'lww_conflict_tbl'"); + last if $on_n2 eq '1'; + sleep(1); +} +is($on_n2, '1', 'lww_conflict_tbl replicated to n2'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "INSERT INTO lww_conflict_tbl (id, val, writer) VALUES (1, 'seed', 'n1')"; + +my $seed_on_n2 = ''; +for (1 .. 30) { + $seed_on_n2 = scalar_query(2, + "SELECT val FROM lww_conflict_tbl WHERE id = 1"); + last if $seed_on_n2 eq 'seed'; + sleep(1); +} +is($seed_on_n2, 'seed', 'seed row replicated to n2 before the join starts'); + +# check_preconditions() rejects the join outright if n1's outbound slot to +# an existing peer (n2) has not fully drained -- the DDL/data that set up +# lww_conflict_tbl and its seed row just above leaves exactly that kind of +# residual lag for a moment. Wait for it to clear before launching, or the +# join dies immediately with "source has unreplicated changes pending to an +# existing peer" (confirmed happening in practice). +for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + last if defined $lag && $lag eq '0'; + sleep(1); +} + +# Log every conflict resolution spock makes to spock.resolutions on n1/n2, +# so the LWW decision can be inspected directly rather than only inferred +# from the converged value. PGC_SIGHUP -- ALTER SYSTEM + reload is enough, +# no restart needed. +for my $port (@$node_ports) { + system_or_bail "$pg_bin/psql", '-p', $port, '-d', $dbname, '-c', + "ALTER SYSTEM SET spock.save_resolutions = on"; + system_or_bail "$pg_bin/psql", '-p', $port, '-d', $dbname, '-c', + "SELECT pg_reload_conf()"; +} + +# ============================================================================= +# TEST: join n3 via n1 while conflicting writes race on n1 and n2 +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_lww'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.lww'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "spock.save_resolutions=on\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_lww.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '600', +); + +# Forwarding (forward_origins := '{all}' on the catchup subscription) is +# live from "Creating catchup subscription" (Step 14/15) through the end of +# the coverage barrier (Step 20), cleared only by clear_forwarding() just +# before Step 20.6 enables the direct peer subscription -- see +# spock_bidirectional_implementation.md. Waiting for the catchup wait-loop +# log line guarantees we are somewhere inside that whole window, not +# necessarily "still waiting" specifically (with only one seed row, Step 18 +# may already be satisfied): either way, both conflicting writes below are +# still guaranteed to reach n3 via forwarding, not via a live direct +# subscription to n2 (which does not exist yet). +ok(wait_for_log_pattern($scs_log, qr/Waiting for catchup to the source/, 60), + 'join reached the catchup wait phase (forwarding is live)') + or BAIL_OUT('spock_create_subscriber never reached catchup; see ' . $scs_log); + +# Fire both conflicting UPDATEs as near-simultaneously as two forked OS +# processes allow, so neither is sequenced through the other via the +# existing n1<->n2 mesh subscription before the second one commits. +my $writer1_log = "$log_dir/lww_writer_n1.log"; +my $writer2_log = "$log_dir/lww_writer_n2.log"; +my $w1_pid = spawn_background($writer1_log, + "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, '-c', + "UPDATE lww_conflict_tbl SET val = 'from_n1', writer = 'n1' WHERE id = 1"); +my $w2_pid = spawn_background($writer2_log, + "$pg_bin/psql", '-X', '-p', $node_ports->[1], '-d', $dbname, '-c', + "UPDATE lww_conflict_tbl SET val = 'from_n2', writer = 'n2' WHERE id = 1"); + +my $w1_rc = wait_for_pid($w1_pid, 30); +my $w2_rc = wait_for_pid($w2_pid, 30); +is($w1_rc, 0, 'conflicting UPDATE from n1 committed') + or diag("see $writer1_log"); +is($w2_rc, 0, 'conflicting UPDATE from n2 committed') + or diag("see $writer2_log"); + +my $scs_rc = wait_for_pid($scs_pid, 780); +unless (defined $scs_rc) { + diag("spock_create_subscriber (lww) did not exit within 780s; killing it"); + kill('TERM', $scs_pid); + waitpid($scs_pid, 0); + $scs_rc = -1; +} +is($scs_rc, 0, '--bidirectional exits 0 despite a same-row conflict during catchup') + or diag("see $scs_log"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3'); +ok(wait_for_sub_status(3, 'sub_n3_n2', 'replicating', 30), + 'direct peer subscription sub_n3_n2 is replicating on n3'); + +# ============================================================================= +# THE assertion: all three nodes converge on the SAME LWW winner. n1 and n2 +# resolve the conflict live, in real time, via their existing mesh +# subscription; n3 only ever sees it via forwarding through n1's catchup +# subscription. If forwarding lost the origin tag or reordered the two +# writes, n3 would independently resolve to a different winner than n1/n2 +# even though row counts and the rest of the join look perfectly healthy. +# ============================================================================= +my $val_n1 = ''; +my $val_n2 = ''; +my $val_n3 = ''; +for (1 .. 60) { + $val_n1 = scalar_query(1, "SELECT val FROM lww_conflict_tbl WHERE id = 1"); + $val_n2 = scalar_query(2, "SELECT val FROM lww_conflict_tbl WHERE id = 1"); + $val_n3 = scalar_query(3, "SELECT val FROM lww_conflict_tbl WHERE id = 1"); + last if $val_n1 eq $val_n2 && $val_n2 eq $val_n3; + sleep(1); +} +diag("final values -- n1: '$val_n1', n2: '$val_n2', n3: '$val_n3'"); + +ok($val_n1 eq 'from_n1' || $val_n1 eq 'from_n2', + 'n1 resolved the conflict to one of the two contending writers, not a third value'); +is($val_n2, $val_n1, 'n2 agrees with n1 on the LWW winner (both saw the conflict live)'); +is($val_n3, $val_n1, + 'n3 agrees with n1/n2 on the LWW winner despite only seeing it forwarded'); + +my $exc_count_n3 = scalar_query(3, "SELECT count(*) FROM spock.exception_log"); +is($exc_count_n3, '0', 'no exceptions logged on n3 for the resolved conflict'); + +# ============================================================================= +# Best-effort: confirm the resolution was actually logged (spock.resolutions), +# not just inferred from the converged value. Not load-bearing for the core +# assertion above -- resolution logging is an independent code path +# (spock_conflict.c, spock_save_resolutions) from the value that ends up on +# disk, so its absence would be a separate, secondary finding. +my $resolutions_n1 = '0'; +for (1 .. 30) { + $resolutions_n1 = scalar_query(1, + "SELECT count(*) FROM spock.resolutions WHERE relname = 'lww_conflict_tbl'"); + last if $resolutions_n1 ne '0'; + sleep(1); +} +if ($resolutions_n1 ne '0') { + pass('n1 logged a conflict resolution for lww_conflict_tbl to spock.resolutions'); + my $conflict_type = scalar_query(1, + "SELECT conflict_type FROM spock.resolutions " . + "WHERE relname = 'lww_conflict_tbl' ORDER BY id LIMIT 1"); + like($conflict_type, qr/UPDATE_ORIGIN_DIFFERS/, + 'logged conflict_type matches a same-row concurrent update'); +} else { + diag('no spock.resolutions row found on n1 for lww_conflict_tbl within 30s ' . + '(informational only -- not required for this scenario)'); +} + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/tests/tap/t/056_bidir_join_concurrent_load.pl b/tests/tap/t/056_bidir_join_concurrent_load.pl new file mode 100644 index 00000000..669bc6b9 --- /dev/null +++ b/tests/tap/t/056_bidir_join_concurrent_load.pl @@ -0,0 +1,381 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 056_bidir_join_concurrent_load.pl - spock_create_subscriber +# --bidirectional under simultaneous +# write load on the SOURCE and a +# non-source PEER at once +# ============================================================================= +# Design doc (spock_bidirectional_final.md) section 16 lists, as a required +# positive-path scenario: "Concurrent writes on source AND a non-source peer +# throughout catchup; assert row-level convergence on all nodes (no missing +# rows, no duplicates)." Two existing tests each cover half of this: +# 049_bidir_join_under_load.pl loads only the source (n1); 051_bidir_join_ +# multipeer_lag.pl loads only a peer (n3, in a 3-node mesh). Neither loads +# both at once, which is what the design scenario actually calls for -- a +# bug specific to two independent write streams converging through the +# forwarding path simultaneously (e.g. an interleaving that the barrier or +# forwarded-origin resolution handles correctly when only one side is active) +# would not be caught by either. +# +# This test keeps the topology as simple as the scenario allows: a 2-node +# pre-existing mesh (n1, n2), joining n3 via n1 as the source -- n1 is "the +# source", n2 is "a non-source peer", matching the design wording exactly +# with no extra nodes needed. Uses the same dependency-free write-load +# mechanism as 051 (a PROCEDURE issuing its own COMMITs per batch, driven by +# a backgrounded psql -c "CALL ...", throttled with pg_sleep) rather than +# 049's pgedge-loadgen dependency -- keeping this test self-contained and +# suitable for the per-push schedule, not gated behind external tooling. +# +# Each writer inserts into disjoint id ranges (n1: 1..999999, n2: 50000000+) +# so a real bug that produced a genuine duplicate row (not just a benign +# conflict-resolution replay) is distinguishable from two writers legitimately +# both succeeding: convergence is checked via an EXACT row-count match (not +# ">="), a COUNT(DISTINCT id) = COUNT(*) uniqueness check, and a full content +# hash, on every node, against both writers' rows at once. +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok get_test_config scalar_query + wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally (n1 <-> n2) +# ============================================================================= +create_cluster(2, 'Create 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2'); + +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out //= ''; + $out =~ s/^\s+|\s+$//g; + return $out; +} + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +sub wait_for_table_drain { + my ($node_a, $node_b, $table, $timeout) = @_; + $timeout //= 180; + my $count_a = scalar_query($node_a, "SELECT COUNT(*) FROM $table"); + for (1 .. $timeout) { + my $count_b = scalar_query($node_b, "SELECT COUNT(*) FROM $table"); + return 1 if defined $count_b && $count_b eq $count_a; + sleep(1); + } + return 0; +} + +# ============================================================================= +# SETUP: a table + PROCEDURE replicated across n1/n2 via DDL replication, +# used by both concurrent writers below. Disjoint id ranges per writer (a +# large offset for n2) rather than a shared sequence -- two independent +# writers sharing one sequence would themselves need to replicate sequence +# state through the very path being tested, entangling the setup with the +# thing under test. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE concurrent_load_tbl (id bigint PRIMARY KEY, val text, src text)"; +pass('concurrent_load_tbl created on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', q{ + CREATE PROCEDURE concurrent_load(id_base bigint, n_batches int, + batch_rows int, node_label text) + LANGUAGE plpgsql AS $$ + DECLARE i int; + BEGIN + FOR i IN 1..n_batches LOOP + INSERT INTO concurrent_load_tbl (id, val, src) + SELECT id_base + (i - 1) * batch_rows + g, 'x' || g, node_label + FROM generate_series(1, batch_rows) g; + COMMIT; + -- Throttled for the same reason as 051_bidir_join_multipeer_ + -- lag.pl's writer: both n1 and n2 apply each other's rows via + -- their pre-existing cross-wired subscription, so an unthrottled + -- writer on either side inflates both nodes' outbound WAL, not + -- just its own. + PERFORM pg_sleep(0.02); + END LOOP; + END $$; +}; +pass('concurrent_load procedure created on n1'); + +my $table_on_n2 = '0'; +for (1 .. 30) { + $table_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'concurrent_load_tbl'"); + last if $table_on_n2 eq '1'; + sleep(1); +} +is($table_on_n2, '1', 'concurrent_load_tbl replicated to n2'); + +my $proc_on_n2 = '0'; +for (1 .. 30) { + $proc_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_proc WHERE proname = 'concurrent_load'"); + last if $proc_on_n2 eq '1'; + sleep(1); +} +is($proc_on_n2, '1', 'concurrent_load procedure replicated to n2'); + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1 outbound replication never drained after setup; cannot proceed'); + +# ============================================================================= +# TEST: join n3 via n1 while BOTH n1 (the source) and n2 (a non-source peer) +# are under concurrent, independent write load throughout catchup and the +# coverage barrier. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_concurrent'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +sub wait_for_slot_created { + my ($timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $n3_pending || -f $n3_manifest; + sleep(1); + } + return 0; +} + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.concurrent'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_concurrent.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '600', +); + +ok(wait_for_slot_created(30), + 'source slot created (safe to start both writers)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); + +# Two independent writers, started together: one against the source (n1), +# one against a non-source peer (n2), both left running through catchup and +# the coverage barrier. +my $n1_writer_pid = spawn_background("$log_dir/concurrent_load_n1.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, + '-c', "CALL concurrent_load(1, 400, 50, 'n1')"); +my $n2_writer_pid = spawn_background("$log_dir/concurrent_load_n2.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[1], '-d', $dbname, + '-c', "CALL concurrent_load(50000000, 400, 50, 'n2')"); +pass('concurrent writers started on n1 (source) and n2 (peer)'); + +my $scs_rc = wait_for_pid($scs_pid, 780); +unless (defined $scs_rc) { + diag("spock_create_subscriber did not exit within 780s; killing it"); + kill('TERM', $scs_pid); + waitpid($scs_pid, 0); + $scs_rc = -1; +} +is($scs_rc, 0, + '--bidirectional exits 0 with concurrent source+peer write load throughout') + or diag("see $scs_log"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3'); +ok(wait_for_sub_status(3, 'sub_n3_n2', 'replicating', 30), + 'direct peer subscription sub_n3_n2 is replicating on n3'); +ok(wait_for_sub_status(2, 'sub_n2_n3', 'replicating', 30), + 'reverse subscription sub_n2_n3 is replicating on n2'); +ok(wait_for_sub_status(1, 'sub_n1_n3', 'replicating', 30), + 'reverse subscription sub_n1_n3 is replicating on n1'); + +my $readonly = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SHOW spock.readonly"); +is($readonly, 'off', 'spock.readonly is lifted on n3'); + +# ============================================================================= +# Let both writers finish naturally (each does a fixed, bounded number of +# batches), then let replication fully drain everywhere before comparing. +# ============================================================================= +sub stop_writer { + my ($pid, $timeout) = @_; + return unless defined $pid; + my $rc = wait_for_pid($pid, $timeout); + unless (defined $rc) { + diag("writer pid $pid did not finish within ${timeout}s; killing it"); + kill('TERM', $pid); + waitpid($pid, 0); + } +} +stop_writer($n1_writer_pid, 120); +stop_writer($n2_writer_pid, 120); +pass('both writers finished'); + +diag('concurrent_load_tbl did not fully drain to n3 within 180s') + unless wait_for_table_drain(1, 3, 'concurrent_load_tbl'); +diag('concurrent_load_tbl did not fully drain to n2 within 180s') + unless wait_for_table_drain(1, 2, 'concurrent_load_tbl'); + +# ============================================================================= +# THE assertions: every node must agree exactly -- no missing rows (either +# writer's), no duplicates -- on the combined output of both simultaneous +# writers. +# ============================================================================= +my $expected_total = 400 * 50 * 2; # n_batches * batch_rows, both writers +my $hash_query = "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) " . + "FROM concurrent_load_tbl x"; + +for my $pair ([1, 'n1'], [2, 'n2'], [3, 'n3']) { + my ($node_num, $label) = @$pair; + my $count = scalar_query($node_num, "SELECT COUNT(*) FROM concurrent_load_tbl"); + is($count, $expected_total, + "$label has exactly the expected row count from both writers (no missing rows)"); + my $distinct_count = scalar_query($node_num, + "SELECT COUNT(DISTINCT id) FROM concurrent_load_tbl"); + is($distinct_count, $expected_total, + "$label has no duplicate ids (COUNT(DISTINCT id) matches COUNT(*))"); +} + +my $hash_n1 = scalar_query(1, $hash_query); +my $hash_n2 = scalar_query(2, $hash_query); +my $hash_n3 = scalar_query(3, $hash_query); +is($hash_n2, $hash_n1, 'content hash matches between n1 and n2'); +is($hash_n3, $hash_n1, 'content hash matches between n1 (source) and n3 (joined)'); +is($hash_n3, $hash_n2, 'content hash matches between n2 (peer) and n3 (joined)'); + +my $src_n1_on_n3 = scalar_query(3, + "SELECT COUNT(*) FROM concurrent_load_tbl WHERE src = 'n1'"); +is($src_n1_on_n3, 400 * 50, "n3 has all of the source's (n1) rows"); +my $src_n2_on_n3 = scalar_query(3, + "SELECT COUNT(*) FROM concurrent_load_tbl WHERE src = 'n2'"); +is($src_n2_on_n3, 400 * 50, "n3 has all of the peer's (n2) rows"); + +# ============================================================================= +# Fresh bidirectional dataflow, proving the mesh is genuinely live after +# both simultaneous write streams and the join itself have settled. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', + "INSERT INTO concurrent_load_tbl (id, val, src) " . + "VALUES (999999999, 'post_join', 'n3')"; +my $marker_on_n1 = '0'; +for (1 .. 30) { + $marker_on_n1 = scalar_query(1, + "SELECT COUNT(*) FROM concurrent_load_tbl WHERE val = 'post_join'"); + last if $marker_on_n1 eq '1'; + sleep(1); +} +is($marker_on_n1, '1', 'fresh write on n3 reaches n1 after the join'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/tests/tap/t/057_bidir_join_stall_abort.pl b/tests/tap/t/057_bidir_join_stall_abort.pl new file mode 100644 index 00000000..9b55837c --- /dev/null +++ b/tests/tap/t/057_bidir_join_stall_abort.pl @@ -0,0 +1,456 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 057_bidir_join_stall_abort.pl - the catchup stall watchdog must +# actually fire, cleanly, when +# progress genuinely stops +# ============================================================================= +# Design doc (spock_bidirectional_final.md) section 16, progress-watchdog +# scenarios: "Stall a wait (freeze *all* progress signals) past +# --stall-timeout -> clean abort with an attributed bottleneck message; +# state preserved; --cleanup recovers; retry resumes." 051_bidir_join_ +# multipeer_lag.pl and 052_bidir_join_big_txn_liveness.pl already prove the +# two "must NOT abort" halves of the same watchdog (slow-but-progressing, +# and a healthy worker busy on one large transaction) -- but nothing +# exercised the abort path itself actually firing. Confirmed by reading +# wait_for_origin_progress() in spock_create_subscriber.c: a stall is only +# declared when BOTH origin LSN progress is frozen AND (for a caller with a +# watch_sub_name) apply_worker_is_busy() reports the apply worker is not +# active -- so to hit this deterministically, both signals need to freeze +# at once, not just one. +# +# This freezes both signals the same way a real network partition or dead +# link would: rather than manufacturing a scenario through extra load or +# timing, it SIGSTOPs the actual walsender backend on n1 serving n3's +# catchup subscription. Found precisely, not "most recently connected" (to +# avoid any ambiguity with the pre-existing n1<->n2 mesh's own apply +# connections): spock_connect_replica() (spock.c) passes the subscription's +# own slot_name as the outbound connection's application_name, so on n1 +# that walsender's pg_stat_replication.application_name is exactly +# sub_n3_n1's sub_slot_name. (This is a different application_name than +# the "spock apply :" convention apply_worker_is_busy() uses +# -- that one is set locally on n3's own backend via SetConfigOption() in +# spock_worker.c, for n3's own pg_stat_activity; it is never sent to n1.) +# A stopped walsender sends nothing further, so n3's origin LSN stops +# advancing, and once the apply worker finishes whatever was already in +# flight it goes idle -- both conditions the watchdog requires, +# deterministically and without touching any implementation code. +# +# After the abort: SIGCONT is required before --cleanup, since DROP +# ... REPLICATION SLOT waits for its walsender to detach, which a stopped +# backend never does. --cleanup --force must then recover cleanly, the +# pre-existing n1<->n2 mesh must be unaffected, and a fresh retry (against +# the now-unstalled source) must succeed end-to-end -- the same "genuinely +# rejoinable, not just cleaned up" bar 050_bidir_join_crash_midcatchup.pl +# sets for its own crash scenario. +# +# Unix-only (SIGSTOP/SIGCONT process control), consistent with the rest of +# this test family's existing fork()/exec()/kill() use for spawning and +# killing spock_create_subscriber itself. +# ============================================================================= + +use strict; +use warnings; +use Test::More; +use File::Path qw(remove_tree); +use POSIX qw(:sys_wait_h); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready wait_for_sub_status); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; +my $log_dir = $config->{log_dir}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +sub spawn_background { + my ($logfile, @cmd) = @_; + my $pid = fork(); + die "fork() failed: $!" unless defined $pid; + if ($pid == 0) { + open(my $fh, '>>', $logfile) or die "Cannot open $logfile: $!"; + open(STDOUT, '>&', $fh) or die $!; + open(STDERR, '>&', $fh) or die $!; + close($fh); + exec(@cmd) or exit(127); + } + return $pid; +} + +sub wait_for_pid { + my ($pid, $timeout) = @_; + for (1 .. $timeout) { + my $r = waitpid($pid, WNOHANG); + return ($? >> 8) if $r == $pid; + sleep(1); + } + return undef; +} + +sub wait_for_zero_lag { + my ($node_num, $timeout) = @_; + for (1 .. $timeout) { + my $lag = scalar_query($node_num, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + return 1 if defined $lag && $lag eq '0'; + sleep(1); + } + return 0; +} + +sub wait_for_slot_created { + my ($pending_path, $manifest_path, $timeout) = @_; + for (1 .. $timeout) { + return 1 if -f $pending_path || -f $manifest_path; + sleep(1); + } + return 0; +} + +# Poll a log file's content for a pattern (rather than a fixed offset read) +# since the file may not exist yet when polling starts. +sub wait_for_log_pattern { + my ($logfile, $pattern, $timeout) = @_; + for (1 .. $timeout) { + if (-f $logfile) { + open(my $fh, '<', $logfile) or die "Cannot open $logfile: $!"; + local $/; + my $content = <$fh>; + close($fh); + return 1 if defined $content && $content =~ $pattern; + } + sleep(1); + } + return 0; +} + +sub slurp_log { + my ($logfile) = @_; + return '' unless -f $logfile; + open(my $fh, '<', $logfile) or die "Cannot open $logfile: $!"; + local $/; + my $content = <$fh>; + close($fh); + return $content // ''; +} + +# ============================================================================= +# SETUP: a continuous writer on n1, so catchup always has a real, moving +# backlog right up until the freeze -- same rationale as +# 050_bidir_join_crash_midcatchup.pl: a one-time seed already present in +# the physical backup leaves nothing for the logical catchup subscription +# to replay, so only rows written *during* the join are real catchup work. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE stall_test_tbl (id bigint PRIMARY KEY, val text)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE stall_test_id_seq"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', q{ + CREATE PROCEDURE stall_test_load(n_batches int, batch_rows int) + LANGUAGE plpgsql AS $$ + DECLARE i int; + BEGIN + FOR i IN 1..n_batches LOOP + INSERT INTO stall_test_tbl + SELECT nextval('stall_test_id_seq'), 'x' || g + FROM generate_series(1, batch_rows) g; + COMMIT; + PERFORM pg_sleep(0.02); + END LOOP; + END $$; +}; +pass('stall_test_tbl and writer procedure created on n1'); + +ok(wait_for_zero_lag(1, 60), 'replication drained before starting the join') + or BAIL_OUT('n1->n2 replication never drained after setup; cannot proceed'); + +my $writer_pid; + +sub start_writer { + $writer_pid = spawn_background("$log_dir/stall_test_writer.log", + "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, + '-c', "CALL stall_test_load(100000, 500)"); +} + +sub stop_writer { + system_or_bail "$pg_bin/psql", '-X', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " . + "WHERE query LIKE 'CALL stall_test_load%' AND pid <> pg_backend_pid()"; + for (1 .. 30) { + my $still_running = scalar_query(1, + "SELECT count(*) FROM pg_stat_activity WHERE query LIKE 'CALL stall_test_load%'"); + last if defined $still_running && $still_running eq '0'; + sleep(1); + } + kill('TERM', $writer_pid); + waitpid($writer_pid, 0); +} + +# ============================================================================= +# TEST: start --bidirectional, freeze the source walsender partway through +# the join (both LSN progress and apply-worker liveness at once -- this +# lands during catchup or, if catchup finishes first against this test's +# small backlog, the immediately-following coverage barrier's Hop 2, since +# both poll the same sub_n3_n1 connection), and confirm the watchdog +# aborts cleanly with an attributed message. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_stall'; +my $n3_pending = "${n3_datadir}.spock_bidir_pending.json"; +my $n3_manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; +my $stall_timeout = 5; + +remove_tree($n3_datadir) if -d $n3_datadir; +unlink($n3_pending) if -f $n3_pending; + +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override.stall'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='$log_dir'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +my $scs_log = "$log_dir/scs_stall_abort.log"; +unlink($scs_log) if -f $scs_log; + +my $scs_pid = spawn_background($scs_log, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', "$stall_timeout", + '--max-wait', '300', +); + +ok(wait_for_slot_created($n3_pending, $n3_manifest, 30), + 'source slot created (safe to start the writer)') + or BAIL_OUT('spock_create_subscriber never created the source slot; see ' . $scs_log); +start_writer(); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 60), + 'n3 postgres is running (pre-freeze)') + or BAIL_OUT('n3 postgres never came up; see ' . $scs_log); + +ok(wait_for_log_pattern($scs_log, qr/Waiting for catchup to the source/, 120), + 'spock_create_subscriber reached the catchup wait phase') + or BAIL_OUT('never reached the catchup wait phase; see ' . $scs_log); + +stop_writer(); + +# Precisely identify n3's own catchup-subscription apply connection on n1 +# -- not "most recently connected" -- via the slot name spock_connect_ +# replica() uses as that connection's application_name, so there is no +# ambiguity with the pre-existing n1<->n2 mesh's own apply connections. +my $n3_slot_name = scalar_query(3, + "SELECT sub_slot_name FROM spock.subscription WHERE sub_name = 'sub_n3_n1'"); +ok(defined $n3_slot_name && length($n3_slot_name), 'read sub_n3_n1 slot name') + or BAIL_OUT('could not read sub_n3_n1 slot name; see ' . $scs_log); + +my $walsender_pid = scalar_query(1, + "SELECT pid FROM pg_stat_replication WHERE application_name = '$n3_slot_name'"); +ok(defined $walsender_pid && $walsender_pid =~ /^\d+$/, + "found n1's walsender backend for sub_n3_n1 (pid $walsender_pid)") + or BAIL_OUT("could not find walsender for application_name '$n3_slot_name'"); + +ok(kill('STOP', $walsender_pid), "SIGSTOPped n1's walsender for sub_n3_n1") + or BAIL_OUT("could not SIGSTOP walsender pid $walsender_pid: $!"); + +my $scs_rc = wait_for_pid($scs_pid, 30); + +# Always resume the frozen backend before doing anything else, regardless +# of how the assertions below turn out -- a stopped walsender left behind +# would otherwise wedge every later step in this test (including +# --cleanup's own slot drop, and Test::More's own teardown). +ok(kill('CONT', $walsender_pid), "SIGCONTed n1's walsender for sub_n3_n1"); + +ok(defined $scs_rc, 'spock_create_subscriber exited on its own (watchdog fired)') + or diag("spock_create_subscriber did not exit within 30s after the freeze; see $scs_log"); +if (!defined $scs_rc) { + kill('TERM', $scs_pid); + waitpid($scs_pid, 0); + $scs_rc = -1; +} +isnt($scs_rc, 0, + 'spock_create_subscriber exits non-zero once the stall watchdog fires'); + +my $scs_output = slurp_log($scs_log); +like($scs_output, + qr/\S[^\n]*? appears stalled: no origin progress for \d+ second\(s\) and the apply worker is not active \(--stall-timeout\)/, + 'abort message attributes the stall to a specific phase, not a generic wait'); + +# ============================================================================= +# TEST: --cleanup --force recovers cleanly after the aborted join +# ============================================================================= +# die()'s own handler (spock_create_subscriber.c) already tries "pg_ctl +# stop -s" on n3 before exiting. Depending on exactly how far that +# shutdown has progressed by the time --cleanup runs moments later, +# cleanup_partial_state() may or may not still be able to reach n3 for a +# fully "confirmed" cleanup -- if not, it deliberately returns non-zero +# and prints "Cleanup incomplete ... keeping the manifest/sidecar record +# so --cleanup can be retried", an explicit invitation to run it again, +# not a failure to route around. Both outcomes are legitimate depending +# on timing, so this exercises the documented retry contract rather than +# assuming either one. +sub run_scs_capture { + my ($logfile, @args) = @_; + unlink($logfile) if -f $logfile; + my $pid = spawn_background($logfile, $SCS_BIN, @args); + my $rc = wait_for_pid($pid, 60); + unless (defined $rc) { + kill('TERM', $pid); + waitpid($pid, 0); + $rc = -1; + } + return $rc; +} + +my $cleanup_log1 = "$log_dir/scs_stall_abort_cleanup1.log"; +my $cleanup_rc1 = run_scs_capture($cleanup_log1, + '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir); + +if ($cleanup_rc1 == 0) { + pass('--cleanup --force exits 0 after the stall abort (first attempt)'); +} else { + like(slurp_log($cleanup_log1), + qr/Cleanup incomplete.*keeping the manifest.*retried/is, + 'first cleanup attempt reports the documented "retry" outcome, ' . + 'not an unrelated failure') + or diag("see $cleanup_log1"); + + my $cleanup_log2 = "$log_dir/scs_stall_abort_cleanup2.log"; + my $cleanup_rc2 = run_scs_capture($cleanup_log2, + '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir); + is($cleanup_rc2, 0, + '--cleanup --force succeeds on retry, per its own documented contract') + or diag("see $cleanup_log2"); +} + +my $source_slot_after_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after_cleanup, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); + +my $n1n2_ok_after_stall = wait_for_sub_status(1, 'sub_n1_n2', 'replicating', 30) + && wait_for_sub_status(2, 'sub_n2_n1', 'replicating', 30); +ok($n1n2_ok_after_stall, + 'pre-existing n1 <-> n2 mesh is unaffected by the stall + abort + cleanup cycle'); + +# ============================================================================= +# TEST: a fresh --bidirectional retry against the now-unstalled source +# succeeds end-to-end -- the cluster is genuinely rejoinable, not just +# "cleaned up". +# ============================================================================= +my $final_count = scalar_query(1, "SELECT COUNT(*) FROM stall_test_tbl"); +ok(defined $final_count && $final_count > 0, + "writer produced $final_count rows on n1 before it was stopped"); + +ok(wait_for_zero_lag(1, 60), 'replication drained before the retry') + or BAIL_OUT('n1->n2 replication never drained after cleanup; cannot retry'); + +my $scs_log_retry = "$log_dir/scs_stall_abort_retry.log"; +unlink($scs_log_retry) if -f $scs_log_retry; + +my $scs_pid_retry = spawn_background($scs_log_retry, + $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + '--stall-timeout', '120', + '--max-wait', '300', +); + +my $scs_rc_retry = wait_for_pid($scs_pid_retry, 300); +unless (defined $scs_rc_retry) { + diag("spock_create_subscriber (retry) did not exit within 300s; killing it"); + kill('TERM', $scs_pid_retry); + waitpid($scs_pid_retry, 0); + $scs_rc_retry = -1; +} +is($scs_rc_retry, 0, 'fresh --bidirectional retry after the stall abort exits 0') + or diag("see $scs_log_retry"); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running (retry)'); +ok(wait_for_sub_status(3, 'sub_n3_n1', 'replicating', 30), + 'catchup subscription sub_n3_n1 is replicating on n3 (retry)'); + +my $n3_count_retry = '-1'; +for (1 .. 60) { + $n3_count_retry = scalar_query(3, "SELECT COUNT(*) FROM stall_test_tbl"); + last if $n3_count_retry eq $final_count; + sleep(1); +} +is($n3_count_retry, $final_count, 'all rows present on n3 after the retry'); + +my $hash_n1 = scalar_query(1, + "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) FROM stall_test_tbl x"); +my $hash_n3 = scalar_query(3, + "SELECT md5(COALESCE(string_agg(x::text, ',' ORDER BY id), '')) FROM stall_test_tbl x"); +is($hash_n3, $hash_n1, 'content hash matches between n1 and n3 after the retry'); + +command_ok( + [ $SCS_BIN, '--bidirectional', '--cleanup', '--force', '--pgdata', $n3_datadir ], + '--cleanup --force (final teardown) exits 0' +); +ok(!-d $n3_datadir, 'n3 data directory removed after final cleanup'); + +# ============================================================================= +# CLEANUP +# ============================================================================= +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Destroy 2-node cluster'); + +done_testing(); diff --git a/utils/pgindent/typedefs.list b/utils/pgindent/typedefs.list index 298212b0..1a38eccb 100644 --- a/utils/pgindent/typedefs.list +++ b/utils/pgindent/typedefs.list @@ -1,13 +1,9 @@ -ACCESS_ALLOWED_ACE -ACL -ACL_SIZE_INFORMATION -AFFIX -ASN1_INTEGER -ASN1_OBJECT -ASN1_OCTET_STRING -ASN1_STRING -ATAlterConstraint -AV +__m128i +__m512i +__mmask64 +_SPI_connection +_SPI_plan +_stringlist A_ArrayExpr A_Const A_Expr @@ -16,6 +12,7 @@ A_Indices A_Indirection A_Star AbsoluteTime +access_vector_t AccessMethodInfo AccessPriv Acl @@ -23,12 +20,17 @@ AclItem AclMaskHow AclMode AclResult +acquireLocksOnSubLinks_context AcquireSampleRowsFunc ActionErrCallbackArg ActionList ActiveSnapshotElt +add_nulling_relids_context +addFkConstraintSides AddForeignUpdateTargets_function AddrInfo +adjust_appendrel_attrs_context +AFFIX AffixNode AffixNodeData AfterTriggerBatchCallback @@ -37,9 +39,9 @@ AfterTriggerEvent AfterTriggerEventChunk AfterTriggerEventData AfterTriggerEventList +AfterTriggersData AfterTriggerShared AfterTriggerSharedData -AfterTriggersData AfterTriggersQueryData AfterTriggersTableData AfterTriggersTransData @@ -48,6 +50,8 @@ AggClauseCosts AggClauseInfo AggInfo AggPath +Aggref +AggregateInstrumentation AggSplit AggState AggStatePerAgg @@ -62,18 +66,17 @@ AggStatePerTrans AggStatePerTransData AggStrategy AggTransInfo -Aggref -AggregateInstrumentation AlenState Alias +AllocateDesc +AllocateDescKind AllocBlock AllocFreeListLink +allocfunc AllocPointer AllocSet AllocSetContext AllocSetFreeList -AllocateDesc -AllocateDescKind AlterCollationStmt AlterDatabaseRefreshCollStmt AlterDatabaseSetStmt @@ -81,6 +84,7 @@ AlterDatabaseStmt AlterDefaultPrivilegesStmt AlterDomainStmt AlterDomainType +AlteredTableInfo AlterEnumStmt AlterEventTrigStmt AlterExtensionContentsStmt @@ -88,10 +92,11 @@ AlterExtensionStmt AlterFdwStmt AlterForeignServerStmt AlterFunctionStmt +AlternativeSubPlan AlterObjectDependsStmt AlterObjectSchemaStmt -AlterOpFamilyStmt AlterOperatorStmt +AlterOpFamilyStmt AlterOwnerStmt AlterPkRiChange AlterPolicyStmt @@ -107,9 +112,6 @@ AlterStatsStmt AlterSubscriptionStmt AlterSubscriptionType AlterSystemStmt -AlterTSConfigType -AlterTSConfigurationStmt -AlterTSDictionaryStmt AlterTableCmd AlterTableMoveAllStmt AlterTablePass @@ -117,12 +119,39 @@ AlterTableSpaceOptionsStmt AlterTableStmt AlterTableType AlterTableUtilityContext +AlterTSConfigType +AlterTSConfigurationStmt +AlterTSDictionaryStmt AlterTypeRecurseParams AlterTypeStmt AlterUserMappingStmt -AlteredTableInfo -AlternativeSubPlan +amadjustmembers_function +ambeginscan_function +ambuild_function +ambuildempty_function +ambuildphasename_function +ambulkdelete_function +amcanreturn_function AmcheckOptions +amcostestimate_function +amendscan_function +amestimateparallelscan_function +amgetbitmap_function +amgettreeheight_function +amgettuple_function +aminitparallelscan_function +aminsert_function +aminsertcleanup_function +ammarkpos_function +amoptions_function +amparallelrescan_function +amproperty_function +amrescan_function +amrestrpos_function +amtranslate_cmptype_function +amtranslate_strategy_function +amvacuumcleanup_function +amvalidate_function AnalyzeAttrComputeStatsFunc AnalyzeAttrFetchFunc AnalyzeForeignTable_function @@ -141,6 +170,9 @@ ApplyReplayEntryData ApplySubXactData Archive ArchiveCheckConfiguredCB +ArchivedWAL_hash +ArchivedWAL_iterator +ArchivedWALFile ArchiveEntryPtrType ArchiveFileCB ArchiveFormat @@ -150,14 +182,13 @@ ArchiveModuleCallbacks ArchiveModuleInit ArchiveModuleState ArchiveOpts +ArchiverOutput +ArchiverStage ArchiveShutdownCB ArchiveStartupCB ArchiveStreamState -ArchivedWALFile -ArchivedWAL_hash -ArchivedWAL_iterator -ArchiverOutput -ArchiverStage +array_iter +array_unnest_fctx ArrayAnalyzeExtraData ArrayBuildState ArrayBuildStateAny @@ -176,91 +207,63 @@ ArraySortCachedInfo ArraySubWorkspace ArrayToken ArrayType +ASN1_INTEGER +ASN1_OBJECT +ASN1_OCTET_STRING +ASN1_STRING +assign_collations_context +astreamer +astreamer_archive_context +astreamer_extractor +astreamer_gzip_decompressor +astreamer_gzip_writer +astreamer_lz4_frame +astreamer_member +astreamer_ops +astreamer_plain_writer +astreamer_recovery_injector +astreamer_tar_archiver +astreamer_tar_parser +astreamer_verify +astreamer_waldump +astreamer_zstd_frame AsyncQueueControl AsyncQueueEntry AsyncRequest +ATAlterConstraint AttInMetadata -AttStatsSlot AttoptCacheEntry AttoptCacheKey -AttrDefInfo AttrDefault +AttrDefInfo +AttributeOpts AttrMap AttrMissing AttrNumber -AttrResultArgMap -AttributeOpts +AttStatsSlot +auth_password_hook_typ AuthRequest AuthToken +auto_explain_extension_options +auto_explain_option AutoPrewarmReadStreamData AutoPrewarmSharedState +autovac_table AutoVacOpts AutoVacuumScores AutoVacuumShmemStruct AutoVacuumWorkItem AutoVacuumWorkItemType AuxProcType -BF_ctx -BF_key -BF_word -BF_word_signed -BIGNUM -BIO -BIO_METHOD -BITVECP -BMS_Comparison -BMS_Membership -BN_CTX -BOOL -BOOLEAN -BOX -BTArrayKeyInfo -BTBuildState -BTCallbackState -BTCycleId -BTDedupInterval -BTDedupState -BTDedupStateData -BTDeletedPageData -BTIndexStat -BTInsertState -BTInsertStateData -BTLeader -BTMetaPageData -BTOneVacInfo -BTOptions -BTPS_State -BTPageOpaque -BTPageOpaqueData -BTPageStat -BTPageState -BTParallelScanDesc -BTPendingFSM -BTReadPageState -BTScanInsert -BTScanInsertData -BTScanKeyPreproc -BTScanOpaque -BTScanOpaqueData -BTScanPos -BTScanPosData -BTScanPosItem -BTShared -BTSortArrayContext -BTSpool -BTStack -BTStackData -BTVacInfo -BTVacState -BTVacuumPosting -BTVacuumPostingData -BTWriteState -BUF_MEM -BYTE -BY_HANDLE_FILE_INFORMATION +AV +av_relation +avc_cache +avl_dbase +avl_node +avl_tree +avw_dbase Backend BackendId -BackendParameters BackendStartupData BackendState BackendType @@ -269,26 +272,68 @@ BackgroundWorker BackgroundWorkerArray BackgroundWorkerHandle BackgroundWorkerSlot +backslashResult +backup_file_entry +backup_file_hash +backup_manifest_info +backup_manifest_option +backup_wal_range BackupState Barrier +base_yy_extra_type +basebackup_options BaseBackupCmd BaseBackupTargetHandle BaseBackupTargetType BasicArchiveData BatchMVCCState +bbsink +bbsink_copystream +bbsink_gzip +bbsink_lz4 +bbsink_ops +bbsink_server +bbsink_shell +bbsink_state +bbsink_throttle +bbsink_zstd +bbstreamer +bbstreamer_archive_context +bbstreamer_extractor +bbstreamer_gzip_decompressor +bbstreamer_gzip_writer +bbstreamer_lz4_frame +bbstreamer_member +bbstreamer_ops +bbstreamer_plain_writer +bbstreamer_recovery_injector +bbstreamer_tar_archiver +bbstreamer_tar_parser +bbstreamer_zstd_frame BeginDirectModify_function BeginForeignInsert_function BeginForeignModify_function BeginForeignScan_function BeginSampleScan_function BernoulliSamplerData -BgWorkerStartTime +BF_ctx +BF_key +BF_word +BF_word_signed BgwHandleStatus +bgworker_main_type +BgWorkerStartTime +bh_node_type +BidirectionalState +BIGNUM BinaryArithmFunc +binaryheap +binaryheap_comparator BinaryUpgradeClassOidItem BindParamCbData +BIO +BIO_METHOD BipartiteMatchState -BitString BitmapAnd BitmapAndPath BitmapAndState @@ -304,13 +349,23 @@ BitmapOr BitmapOrPath BitmapOrState Bitmapset +bitmapword +bits16 +bits32 +bits8 +BitString +BITVECP Block +BlockedProcData +BlockedProcsData BlockId BlockIdData BlockInfoRecord BlockNumber BlockRangeReadStreamPrivate BlockRefTable +blockreftable_hash +blockreftable_iterator BlockRefTableBuffer BlockRefTableChunk BlockRefTableEntry @@ -320,10 +375,9 @@ BlockRefTableSerializedEntry BlockRefTableWriter BlockSampler BlockSamplerData -BlockedProcData -BlockedProcsData BlocksReadStreamData BlocktableEntry +bloom_filter BloomBuildState BloomFilter BloomMetaPageData @@ -336,13 +390,20 @@ BloomScanOpaqueData BloomSignatureWord BloomState BloomTuple +BMS_Comparison +BMS_Membership +BN_CTX BoolAggState +Boolean +BooleanTest BoolExpr BoolExprType +boolKEY BoolTestType -Boolean -BooleanTest +BOX BpChar +brin_column_state +brin_serialize_callback_type BrinBuildState BrinDesc BrinInsertState @@ -359,21 +420,64 @@ BrinSpecialSpace BrinStatsData BrinTuple BrinValues -BtreeCheckState -BtreeLastVisibleEntry -BtreeLevel -Bucket -BufFile -Buffer -BufferAccessStrategy -BufferAccessStrategyType -BufferCacheNumaContext -BufferCacheNumaRec -BufferCacheOsPagesContext -BufferCacheOsPagesRec -BufferCachePagesContext -BufferCachePagesRec -BufferDesc +BTArrayKeyInfo +BTBuildState +BTCallbackState +BTCycleId +BTDedupInterval +BTDedupState +BTDedupStateData +BTDeletedPageData +BTIndexStat +BTInsertState +BTInsertStateData +BTLeader +BTMetaPageData +BTOneVacInfo +BTOptions +BTPageOpaque +BTPageOpaqueData +BTPageStat +BTPageState +BTParallelScanDesc +BTPendingFSM +BTPS_State +BTReadPageState +btree_gin_convert_function +btree_gin_leftmost_function +BtreeCheckState +BtreeLastVisibleEntry +BtreeLevel +BTScanInsert +BTScanInsertData +BTScanKeyPreproc +BTScanOpaque +BTScanOpaqueData +BTScanPos +BTScanPosData +BTScanPosItem +BTShared +BTSortArrayContext +BTSpool +BTStack +BTStackData +BTVacInfo +BTVacState +BTVacuumPosting +BTVacuumPostingData +BTWriteState +Bucket +BUF_MEM +Buffer +BufferAccessStrategy +BufferAccessStrategyType +BufferCacheNumaContext +BufferCacheNumaRec +BufferCacheOsPagesContext +BufferCacheOsPagesRec +BufferCachePagesContext +BufferCachePagesRec +BufferDesc BufferDescPadded BufferHeapTupleTableSlot BufferLockMode @@ -382,6 +486,8 @@ BufferManagerRelation BufferStrategyControl BufferTag BufferUsage +BufFile +build_simple_rel_hook_type BuildAccumulator BuiltinScript BulkInsertState @@ -390,33 +496,10 @@ BulkWriteBuffer BulkWriteState BumpBlock BumpContext +bytea ByteaSortSupport -CACHESIGN CAC_state -CCFastEqualFN -CCHashFN -CEOUC_WAIT_MODE -CFuncHashTabEntry -CHAR -CHECKPOINT -CHKVAL -CIRCLE -CMPDAffix -CONTEXT -COP -CRITICAL_SECTION -CRSSnapshotAction -CState -CTECycleClause -CTEMaterialize -CTESearchClause -CURL -CURLM -CURLMcode -CURLMsg -CURLcode -CURLoption -CV +cached_re_str CachedExpression CachedFunction CachedFunctionCompileCallback @@ -425,38 +508,65 @@ CachedFunctionHashEntry CachedFunctionHashKey CachedPlan CachedPlanSource +CACHESIGN CallContext CallStmt CancelRequestPacket +canonicalize_state Cardinality CaseExpr CaseKind CaseTestExpr CaseWhen Cash +cashKEY CastInfo -CatCInProgress -CatCList -CatCTup -CatCache -CatCacheHeader +CatalogCapture CatalogId +catalogid_hash CatalogIdMapEntry CatalogIndexState +CatCache +CatCacheHeader +CatCInProgress +CatCList +CatCTup +cb_cleanup_dir +cb_options +cb_tablespace +cb_tablespace_mapping +CCFastEqualFN +CCHashFN +CEOUC_WAIT_MODE +CFuncHashTabEntry ChangeContext ChangeVarNodes_callback ChangeVarNodes_context ChannelName +char16_t +char32_t +check_agg_arguments_context +check_function_callback +check_network_data +check_object_relabel_type +check_password_hook_type +check_ungrouped_columns_context CheckPoint -CheckPointStmt -CheckpointStatsData +CHECKPOINT CheckpointerRequest CheckpointerShmemStruct +CheckpointStatsData +CheckPointStmt ChecksumBarrierCondition ChecksumStateType +child_process_kind +CHKVAL +chr Chromosome +CIRCLE CkptSortItem CkptTsStatus +CleanupDropOutcome ClientAuthentication_hook_type ClientCertMode ClientCertName @@ -472,7 +582,10 @@ ClusterInfo ClusterParams ClusterStmt CmdType +CMPDAffix +cmpEntriesArg CoalesceExpr +codes_t CoerceParamHook CoerceToDomain CoerceToDomainValue @@ -481,14 +594,17 @@ CoercionContext CoercionForm CoercionPathType CollAliasData -CollInfo -CollParam CollateClause CollateExpr CollateStrength +collation_cache_entry +collation_cache_hash CollectedATSubcmd CollectedCommand CollectedCommandType +CollInfo +color +colormaprange ColorTrgm ColorTrgmInfo ColumnCompareData @@ -513,9 +629,9 @@ CommitTimestampShared CommonEntry CommonTableExpr CompactAttribute +compare_context CompareScalarsContext CompareType -CompiledExprState CompositeIOData CompositeTypeStmt CompoundAffixFlag @@ -524,38 +640,52 @@ CompressionLocation CompressorState ComputeXidHorizonsResult ConcurrentChangeKind -ConditionVariable -ConditionVariableMinimallyPadded ConditionalStack ConditionalStackData +ConditionVariable +ConditionVariableMinimallyPadded +config_handle +config_var_value ConfigData ConfigVariable ConflictTupleInfo ConflictType +conn_errorMessage_func +conn_oauth_client_id_func +conn_oauth_client_secret_func +conn_oauth_discovery_uri_func +conn_oauth_issuer_id_func +conn_oauth_scope_func +conn_sasl_state_func ConnCacheEntry ConnCacheKey -ConnParams -ConnStatusType -ConnType ConnectionStateEnum -ConnectionTiming ConnectionWarning ConnectionWarningFilter +ConnParams +ConnStatusType +ConnType ConsiderSplitContext Const -ConstrCheck -ConstrType Constraint ConstraintCategory ConstraintInfo ConstraintsSetStmt +ConstrCheck +ConstrType +contain_aggs_of_level_context +contain_placeholder_references_context ControlData ControlFileData -ConvInfo -ConvProcInfo ConversionLocation +convert_testexpr_context ConvertRowtypeExpr +ConvInfo +ConvProcInfo CookedConstraint +COP +copy_data_dest_cb +copy_data_source_cb CopyDest CopyFormat CopyFormatOptions @@ -575,15 +705,25 @@ CopyStmt CopyToRoutine CopyToState CopyToStateData +core_yy_extra_type +core_yyscan_t +core_YYSTYPE +corrupt_items Cost +cost_qual_eval_context CostSelector +count_param_references_context Counters CoverExt CoverPos +cp_hash_func +create_upper_paths_hook_type CreateAmStmt CreateCastStmt CreateConversionStmt +createdb_failure_params CreateDBRelInfo +CreatedbStmt CreateDBStrategy CreateDomainStmt CreateEnumStmt @@ -615,14 +755,27 @@ CreateTableSpaceStmt CreateTransformStmt CreateTrigStmt CreateUserMappingStmt -CreatedbStmt -CredHandle +crosstab_cat_desc +crosstab_HashEnt +CRSSnapshotAction +CState +CTECycleClause CteItem +CTEMaterialize CteScan CteScanState +CTESearchClause CteState CtlCommand -CtxtHandle +CURL +curl_infotype +curl_socket_t +curl_version_info_data +CURLcode +CURLM +CURLMcode +CURLMsg +CURLoption CurrentOfExpr CustomExecMethods CustomOutPtrType @@ -630,25 +783,9 @@ CustomPath CustomScan CustomScanMethods CustomScanState +CV CycleCtr -DBState -DCHCacheEntry -DEADLOCK_INFO -DECountItem -DH -DIR -DNSServiceErrorType -DNSServiceRef -DR_copy -DR_intorel -DR_printtup -DR_sqlfunction -DR_transientrel -DSMREntryType -DSMRegistryCtxStruct -DSMRegistryEntry -DWORD -DWORD64 +DatabaseInfo DataChecksumsStateStruct DataChecksumsWorkerDatabase DataChecksumsWorkerOperation @@ -656,67 +793,96 @@ DataChecksumsWorkerResult DataDirSyncMethod DataDumperPtr DataPageDeleteStack +datapagemap_iterator_t +datapagemap_t DataTypesUsageChecks DataTypesUsageVersionCheck -DatabaseInfo DateADT +dateKEY DateTimeErrorExtra +datetkn Datum DatumTupleFields DbInfo DbInfoArr DbLocaleInfo -DbOidName -DdlOptType -DdlOption -DeClonePtrType +DBState +dce_uuid_t +DCHCacheEntry +dclist_head +DEADLOCK_INFO DeadLockState DeallocateStmt +decimal DeclareCursorStmt +DeClonePtrType DecodedBkpBlock DecodedXLogRecord DecodingOutputState DecodingWorker DecodingWorkerShared +DECountItem +DefaultACLInfo DefElem DefElemAction -DefaultACLInfo DefineStmt DefnDumperPtr DeleteStmt +deparse_columns +deparse_context +deparse_expr_cxt +deparse_namespace DependenciesParseState DependenciesSemanticState DependencyGenerator DependencyGeneratorData DependencyType +derives_hash DeserialIOData DestReceiver -DictISpell +destructor +dev_t +DH DictInt +DictISpell DictSimple DictSnowball DictSubState DictSyn DictThesaurus +digit DimensionInfo +DIR DirectoryMethodData DirectoryMethodFile DisableTimeoutParams +disassembledLeaf DiscardMode DiscardStmt DispatchOption DistanceValue DistinctColInfo DistinctExpr -DoState -DoStmt +dlist_head +dlist_iter +dlist_mutable_iter +dlist_node +dm_letter +dm_node DocRepresentation DomainConstraintCache DomainConstraintRef DomainConstraintState DomainConstraintType DomainIOData +DoStmt +DR_copy +DR_intorel +DR_printtup +DR_sqlfunction +DR_transientrel DropBehavior +DropdbStmt DropOwnedStmt DropReplicationSlotCmd DropRoleStmt @@ -724,39 +890,64 @@ DropStmt DropSubscriptionStmt DropTableSpaceStmt DropUserMappingStmt -DropdbStmt -DumpComponents -DumpCtx -DumpId -DumpOptions -DumpSignalInformation +ds_state +dsa_area +dsa_area_control +dsa_area_pool +dsa_area_span +dsa_handle +dsa_pointer +dsa_pointer_atomic +dsa_segment_header +dsa_segment_index +dsa_segment_map +dshash_compare_function +dshash_copy_function +dshash_hash +dshash_hash_function +dshash_parameters +dshash_partition +dshash_seq_status +dshash_table +dshash_table_control +dshash_table_handle +dshash_table_item +dsm_control_header +dsm_control_item +dsm_handle +dsm_op +dsm_segment +dsm_segment_detach_callback +DSMRegistryCtxStruct +DSMRegistryEntry +DSMREntryType DumpableAcl DumpableObject DumpableObjectType DumpableObjectWithAcl +DumpComponents +DumpCtx +DumpId +DumpOptions +DumpSignalInformation DynamicFileList DynamicZoneAbbrev +EachState +ean13 +eary +EC_KEY +ec_matches_callback_type +ec_member_foreign_arg +ec_member_matches_arg ECDerivesEntry ECDerivesKey -EC_KEY -EDGE -ENGINE -EOM_flatten_into_method -EOM_get_flat_size_method -EPQState -EPlan -EState -EStatus -EVP_CIPHER -EVP_CIPHER_CTX -EVP_MD -EVP_MD_CTX -EVP_PKEY -EachState Edge +EDGE EditableObjectType ElementsState ElidedNode +eLogType +emit_log_hook_type EnableTimeoutParams EndDataPtrType EndDirectModify_function @@ -767,21 +958,28 @@ EndLOPtrType EndLOsPtrType EndOfWalRecoveryInfo EndSampleScan_function +ENGINE EnumItem EolType -EphemeralNameRelationType +EOM_flatten_into_method +EOM_get_flat_size_method EphemeralNamedRelation EphemeralNamedRelationData EphemeralNamedRelationMetadata EphemeralNamedRelationMetadataData +EphemeralNameRelationType +EPQState EquivalenceClass EquivalenceMember EquivalenceMemberIterator ErrorContextCallback ErrorData ErrorSaveContext +EState +EStatus EstimateDSMForeignScan_function EstimationInfo +eval_const_expressions_context EventTriggerCacheEntry EventTriggerCacheItem EventTriggerCacheStateType @@ -789,6 +987,11 @@ EventTriggerData EventTriggerEvent EventTriggerInfo EventTriggerQueryState +EVP_CIPHER +EVP_CIPHER_CTX +EVP_MD +EVP_MD_CTX +EVP_PKEY ExceptionLabelMap ExceptionMap ExecAuxRowMark @@ -809,17 +1012,23 @@ ExecScanRecheckMtd ExecStatus ExecStatusType ExecuteStmt +execution_state ExecutorCheckPerms_hook_type ExecutorEnd_hook_type ExecutorFinish_hook_type ExecutorRun_hook_type ExecutorStart_hook_type +exit_function ExpandedArrayHeader ExpandedObjectHeader ExpandedObjectMethods ExpandedRange ExpandedRecordFieldInfo ExpandedRecordHeader +explain_get_index_name_hook_type +explain_per_node_hook_type +explain_per_plan_hook_type +explain_validate_options_hook_type ExplainDirectModify_function ExplainExtensionOption ExplainForeignModify_function @@ -835,8 +1044,8 @@ ExplainWorkersState ExportedSnapshot Expr ExprContext -ExprContextCallbackFunction ExprContext_CB +ExprContextCallbackFunction ExprDoneCond ExprEvalOp ExprEvalOpLookup @@ -853,60 +1062,85 @@ ExtensionInfo ExtensionLocation ExtensionSiblingCache ExtensionVersionInfo -FDWCollateState -FD_SET -FILE -FILETIME -FPI -FSMAddress -FSMPage -FSMPageData +f_smgr FailoverSlotFilter FailoverSlotFilterKey FakeRelCacheEntry FakeRelCacheEntryData +fasthash_state FastPathMeta FastPathStrongRelationLockData +fd_set +FDWCollateState FdwInfo FdwRoutine +fe_oauth_state +fe_scram_state +fe_scram_state_enum +fetch_range_request FetchDirection FetchDirectionKeywords FetchStmt FieldSelect FieldStore File +FILE +file_action_t +file_content_type_t +file_entry_t +file_type_t FileBackupMethod FileFdwExecutionState FileFdwPlanState +filehash_hash +filehash_iterator +filemap_t FileNameMap FileSet FileTag +fill_string_relopt FilterCommandType FilterObjectType FilterStateData +finalize_primnode_context FinalPathExtraData +find_dependent_phvs_context +find_expr_references_context FindColsContext FindSplitData FindSplitStrat -First +fireRIRonSubLink_context +fix_join_expr_context +fix_scan_expr_context +fix_upper_expr_context +fix_windowagg_cond_context FixedParallelExecutorState FixedParallelState FixedParamState FlagMode +flatten_join_alias_vars_context +flatten_rtes_walker_context Float +float4 +float4KEY +float8 +float8KEY +floating_decimal_32 +floating_decimal_64 FlushPosition +fmAggrefPtr +fmExprContextCallbackFunction +fmgr_hook_type FmgrBuiltin FmgrHookEventType FmgrInfo +fmNodePtr +fmStringInfo ForBothCellState ForBothState ForEachState -ForFiveState -ForFourState -ForPortionOfClause -ForPortionOfExpr -ForPortionOfState -ForThreeState +foreign_glob_cxt +foreign_loc_cxt ForeignAsyncConfigureWait_function ForeignAsyncNotify_function ForeignAsyncRequest_function @@ -920,7 +1154,74 @@ ForeignServer ForeignServerInfo ForeignTable ForeignTruncateInfo +ForFiveState +ForFourState ForkNumber +Form_pg_aggregate +Form_pg_am +Form_pg_amop +Form_pg_amproc +Form_pg_attrdef +Form_pg_attribute +Form_pg_auth_members +Form_pg_authid +Form_pg_cast +Form_pg_class +Form_pg_collation +Form_pg_constraint +Form_pg_conversion +Form_pg_database +Form_pg_db_role_setting +Form_pg_default_acl +Form_pg_depend +Form_pg_enum +Form_pg_event_trigger +Form_pg_extension +Form_pg_foreign_data_wrapper +Form_pg_foreign_server +Form_pg_foreign_table +Form_pg_index +Form_pg_inherits +Form_pg_language +Form_pg_largeobject +Form_pg_largeobject_metadata +Form_pg_namespace +Form_pg_opclass +Form_pg_operator +Form_pg_opfamily +Form_pg_partitioned_table +Form_pg_policy +Form_pg_proc +Form_pg_propgraph_element +Form_pg_propgraph_element_label +Form_pg_propgraph_label +Form_pg_propgraph_label_property +Form_pg_propgraph_property +Form_pg_publication +Form_pg_publication_namespace +Form_pg_publication_rel +Form_pg_range +Form_pg_replication_origin +Form_pg_rewrite +Form_pg_sequence +Form_pg_sequence_data +Form_pg_shdepend +Form_pg_statistic +Form_pg_statistic_ext +Form_pg_statistic_ext_data +Form_pg_subscription +Form_pg_subscription_rel +Form_pg_tablespace +Form_pg_transform +Form_pg_trigger +Form_pg_ts_config +Form_pg_ts_config_map +Form_pg_ts_dict +Form_pg_ts_parser +Form_pg_ts_template +Form_pg_type +Form_pg_user_mapping +FormatNode FormData_pg_aggregate FormData_pg_am FormData_pg_amop @@ -987,72 +1288,12 @@ FormData_pg_type FormData_pg_user_mapping FormData_spock_depend FormExtraData_pg_attribute -Form_pg_aggregate -Form_pg_am -Form_pg_amop -Form_pg_amproc -Form_pg_attrdef -Form_pg_attribute -Form_pg_auth_members -Form_pg_authid -Form_pg_cast -Form_pg_class -Form_pg_collation -Form_pg_constraint -Form_pg_conversion -Form_pg_database -Form_pg_db_role_setting -Form_pg_default_acl -Form_pg_depend -Form_pg_enum -Form_pg_event_trigger -Form_pg_extension -Form_pg_foreign_data_wrapper -Form_pg_foreign_server -Form_pg_foreign_table -Form_pg_index -Form_pg_inherits -Form_pg_language -Form_pg_largeobject -Form_pg_largeobject_metadata -Form_pg_namespace -Form_pg_opclass -Form_pg_operator -Form_pg_opfamily -Form_pg_partitioned_table -Form_pg_policy -Form_pg_proc -Form_pg_propgraph_element -Form_pg_propgraph_element_label -Form_pg_propgraph_label -Form_pg_propgraph_label_property -Form_pg_propgraph_property -Form_pg_publication -Form_pg_publication_namespace -Form_pg_publication_rel -Form_pg_range -Form_pg_replication_origin -Form_pg_rewrite -Form_pg_sequence -Form_pg_sequence_data -Form_pg_shdepend -Form_pg_statistic -Form_pg_statistic_ext -Form_pg_statistic_ext_data -Form_pg_subscription -Form_pg_subscription_rel -Form_pg_tablespace -Form_pg_transform -Form_pg_trigger -Form_pg_ts_config -Form_pg_ts_config_map -Form_pg_ts_dict -Form_pg_ts_parser -Form_pg_ts_template -Form_pg_type -Form_pg_user_mapping -FormatNode +ForPortionOfClause +ForPortionOfExpr +ForPortionOfState +ForThreeState FreeBlockNumberArray +freefunc FreeListData FreePageBtree FreePageBtreeHeader @@ -1061,9 +1302,12 @@ FreePageBtreeLeafKey FreePageBtreeSearchResult FreePageManager FreePageSpanLeader -From FromCharDateMode FromExpr +fsec_t +FSMAddress +FSMPage +FSMPageData FullTransactionId FuncCall FuncCallContext @@ -1080,42 +1324,26 @@ FunctionScan FunctionScanPerFuncState FunctionScanState FuzzyAttrMatchState -GBT_NUMKEY -GBT_NUMKEY_R -GBT_VARKEY -GBT_VARKEY_R -GENERAL_NAME -GISTBuildBuffers -GISTBuildState -GISTDeletedPageContents -GISTENTRY -GISTInsertStack -GISTInsertState -GISTIntArrayBigOptions -GISTIntArrayOptions -GISTNodeBuffer -GISTNodeBufferPage -GISTPageOpaque -GISTPageOpaqueData -GISTPageSplitInfo -GISTSTATE -GISTScanOpaque -GISTScanOpaqueData -GISTSearchHeapItem -GISTSearchItem -GISTTYPE -GIST_SPLITVEC -GMReaderTupleBuffer -GROUP -GUCHashEntry -GV Gather GatherMerge GatherMergePath GatherMergeState GatherPath GatherState +GBT_NUMKEY +GBT_NUMKEY_R +GBT_VARKEY +GBT_VARKEY_R +gbt_vsrt_arg +gbtree_ninfo +gbtree_vinfo Gene +GENERAL_NAME +generate_series_fctx +generate_series_numeric_fctx +generate_series_timestamp_fctx +generate_series_timestamptz_fctx +generate_subscripts_fctx GeneratePruningStepsContext GenerationBlock GenerationContext @@ -1124,6 +1352,10 @@ GenericCosts GenericXLogPageData GenericXLogState GeqoPrivateData +get_attavgwidth_hook_type +get_index_stats_hook_type +get_relation_info_hook_type +get_relation_stats_hook_type GetForeignJoinPaths_function GetForeignModifyBatchSize_function GetForeignPaths_function @@ -1132,7 +1364,8 @@ GetForeignRelSize_function GetForeignRowMarkType_function GetForeignUpperPaths_function GetState -GiSTOptions +gid_t +gin_leafpage_items_state GinBtree GinBtreeData GinBtreeDataLeafInsertData @@ -1169,23 +1402,62 @@ GinTernaryValue GinTuple GinTupleCollector GinVacuumState +ginxlogCreatePostingTree +ginxlogDeleteListPages +ginxlogDeletePage +ginxlogInsert +ginxlogInsertDataInternal +ginxlogInsertEntry +ginxlogInsertListPage +ginxlogRecompressDataLeaf +ginxlogSplit +ginxlogUpdateMeta +ginxlogVacuumDataLeafPage +GIST_SPLITVEC +GISTBuildBuffers GistBuildMode +GISTBuildState +GISTDeletedPageContents +GISTENTRY GistEntryVector GistHstoreOptions GistInetKey +GISTInsertStack +GISTInsertState +GISTIntArrayBigOptions +GISTIntArrayOptions +GISTNodeBuffer +GISTNodeBufferPage GistNSN GistOptBufferingMode +GiSTOptions +GISTPageOpaque +GISTPageOpaqueData +GISTPageSplitInfo +GISTScanOpaque +GISTScanOpaqueData +GISTSearchHeapItem +GISTSearchItem GistSortedBuildLevelState GistSplitUnion GistSplitVector +GISTSTATE GistTsVectorOptions +GISTTYPE GistVacState +gistxlogDelete +gistxlogPage +gistxlogPageDelete +gistxlogPageReuse +gistxlogPageSplit +gistxlogPageUpdate GlobalChannelEntry GlobalChannelKey GlobalTransaction GlobalTransactionData GlobalVisHorizonKind GlobalVisState +GMReaderTupleBuffer GrantRoleOptions GrantRoleStmt GrantStmt @@ -1200,24 +1472,39 @@ Group GroupByColInfo GroupByOrdering GroupClause -GroupPath -GroupPathExtraData -GroupProgressTupDescColumns -GroupResultPath -GroupState -GroupVarInfo +grouping_eqop_callback +grouping_sets_data +grouping_walker_ctx GroupingExprInfo GroupingFunc GroupingSet GroupingSetData GroupingSetKind GroupingSetsPath +GroupPath +GroupPathExtraData +GroupProgressTupDescColumns +GroupResultPath +GroupState +GroupVarInfo +growable_trgm_array +gseg_picksplit_item +gss_buffer_desc +gss_cred_id_t +gss_cred_usage_t +gss_ctx_id_t +gss_key_value_element_desc +gss_key_value_set_desc +gss_name_t +gss_OID_set +gtrgm_consistent_cache GucAction GucBoolAssignHook GucBoolCheckHook GucContext GucEnumAssignHook GucEnumCheckHook +GUCHashEntry GucIntAssignHook GucIntCheckHook GucRealAssignHook @@ -1228,38 +1515,24 @@ GucStack GucStackState GucStringAssignHook GucStringCheckHook +GV +gzFile GzipCompressorState -HANDLE -HASHACTION -HASHBUCKET -HASHCTL -HASHELEMENT -HASHHDR -HASHSEGMENT -HASH_SEQ_STATUS -HE -HEntry -HIST_ENTRY -HKEY -HLOCAL -HMAC_CTX -HMODULE -HOldEntry -HRESULT -HSParser -HSpool -HStore -HTAB -HTSV_Result -HV Hash +HASH_SEQ_STATUS +HASHACTION HashAggBatch HashAggSpill HashAllocFunc +HASHBUCKET HashBuildState HashBulkDeleteStreamPrivate HashCompareFunc HashCopyFunc +HASHCTL +HASHELEMENT +hashfunc +HASHHDR HashIndexStat HashInstrumentation HashJoin @@ -1279,13 +1552,18 @@ HashScanOpaque HashScanOpaqueData HashScanPosData HashScanPosItem +HASHSEGMENT HashSkewBucket HashState HashValueFunc +having_grouping_ctx HbaLine +hbaPort +HE HeadlineJsonState HeadlineParsedText HeadlineWordEntry +heap_page_items_state HeapCheckContext HeapCheckReadStreamData HeapPageFreeze @@ -1299,41 +1577,45 @@ HeapTupleFreeze HeapTupleHeader HeapTupleHeaderData HeapTupleTableSlot +help_handler +HEntry +HIST_ENTRY HistControl +hlCheck +HMAC_CTX +HOldEntry +host_cache_hash HostCacheEntry HostsFileLoadResult HostsLine HotStandbyState +HSParser +HSpool +HStore +hstoreCheckKeyLen_t +hstoreCheckValLen_t +hstorePairs_t +hstoreUniquePairs_t +hstoreUpgrade_t +HTAB +HTSV_Result +HV +hyperLogLogState I32 ICU_Convert_Func -ID -INFIX -INT128 -INTERFACE_INFO -IO -IOContext -IOFuncSelector -IOObject -IOOp -IOStats -IO_STATUS_BLOCK -IPCompareMethod -ITEM -IV -IdentLine IdentifierLookup IdentifySystemCmd +IdentLine IfStackElem +ifState +import_error_callback_arg +ImportForeignSchema_function ImportForeignSchemaStmt ImportForeignSchemaType -ImportForeignSchema_function ImportForeignStatistics_function ImportQual -InProgressEnt -InProgressIO IncludeWal InclusionOpaque -IncrementVarSublevelsUp_context IncrementalBackupInfo IncrementalSort IncrementalSortExecutionStatus @@ -1341,6 +1623,7 @@ IncrementalSortGroupInfo IncrementalSortInfo IncrementalSortPath IncrementalSortState +IncrementVarSublevelsUp_context Index IndexAMProperty IndexAmRoutine @@ -1356,6 +1639,7 @@ IndexClauseSet IndexDeleteCounts IndexDeletePrefetchState IndexDoCheckCallback +indexed_tlist IndexElem IndexFetchHeapData IndexFetchTableData @@ -1379,64 +1663,105 @@ IndexTupleData IndexUniqueCheck IndexVacuumInfo IndxInfo +inet +inet_struct +inetKEY InferClause InferenceElem +INFIX InfoItem InhInfo -InheritableSocket -InitSampleScan_function InitializeDSMForeignScan_function InitializeWorkerForeignScan_function -InjIoErrorState +initRowMethod +InitSampleScan_function InjectionPointCacheEntry InjectionPointCallback InjectionPointCondition InjectionPointConditionType InjectionPointData InjectionPointEntry -InjectionPointSharedState InjectionPointsCtl +InjectionPointSharedState +InjIoErrorState +inline_cte_walker_context +inline_error_callback_arg InlineCodeBlock +ino_t +InProgressEnt +InProgressIO +inquiry InsertStmt +instr_time Instrumentation +int_fast32_t +int_fast64_t +int128 +INT128 Int128AggState +int16 +int16_t +int16KEY +int2vector +int32 +int32_t +int32KEY +int64 +int64_t +int64KEY +int8 +int8_t Int8TransTypeData -IntRBTreeNode Integer IntegerSet InternalDefaultACL InternalGrant +internalPQconninfoOption Interval IntervalAggState +intmax_t IntoClause -InvalMessageArray +intptr_t +IntRBTreeNode +intset_internal_node +intset_leaf_node +intset_node +intvKEY InvalidationInfo InvalidationMsgsGroup -IoMethodOps -IpcMemoryId -IpcMemoryKey -IpcMemoryState -IpcSemaphoreId -IpcSemaphoreKey -IsForeignPathAsyncCapable_function +InvalMessageArray +IO +io_callback_fn +io_stat_col +IOContext +IOFuncSelector +IoMethodOps +IOObject +IOOp +IOStats +IpcMemoryId +IpcMemoryKey +IpcMemoryState +IPCompareMethod +IsForeignPathAsyncCapable_function IsForeignRelUpdatable_function IsForeignScanParallelSafe_function -IsoConnInfo IspellDict Item +ITEM ItemArray ItemId +itemIdCompact +itemIdCompactData ItemIdData ItemPointer ItemPointerData IterateDirectModify_function IterateForeignScan_function IterateJsonStringValuesState +IV JEntry JHashState -JOBOBJECT_BASIC_LIMIT_INFORMATION -JOBOBJECT_BASIC_UI_RESTRICTIONS -JOBOBJECT_SECURITY_LIMIT_INFORMATION JitContext JitInstrumentation JitProviderCallbacks @@ -1445,26 +1770,51 @@ JitProviderInit JitProviderReleaseContextCB JitProviderResetAfterErrorCB Join +join_path_setup_hook_type +join_search_hook_type JoinCostWorkspace JoinDomain JoinExpr JoinHashEntry JoinPath JoinPathExtraData +joinrel_setup_hook_type JoinState JoinTreeItem JoinType JsObject -JsValue +json_aelem_action +json_manifest_error_callback +json_manifest_per_file_callback +json_manifest_per_wal_range_callback +json_manifest_perfile_callback +json_manifest_perwalrange_callback +json_manifest_system_identifier_callback +json_manifest_version_callback +json_ofield_action +json_scalar_action +json_struct_action JsonAggConstructor JsonAggState JsonArgument JsonArrayAgg JsonArrayConstructor JsonArrayQueryConstructor +Jsonb +JsonbAggState JsonBaseObjectInfo +JsonbContainer JsonBehavior JsonBehaviorType +JsonbInState +JsonbIterator +JsonbIteratorToken +JsonbIterState +JsonbPair +JsonbParseState +JsonbSubWorkspace +JsonbTypeCategory +JsonbValue JsonConstructorExpr JsonConstructorExprState JsonConstructorType @@ -1546,111 +1896,32 @@ JsonValueList JsonValueListIterator JsonValueType JsonWrapper -Jsonb -JsonbAggState -JsonbContainer -JsonbInState -JsonbIterState -JsonbIterator -JsonbIteratorToken -JsonbPair -JsonbParseState -JsonbSubWorkspace -JsonbTypeCategory -JsonbValue +JsValue JumbleState JunkFilter KAXCompressReason +keepwal_entry +keepwal_hash +key_t KeyAction KeyActions KeyArray +keyEntryData KeySuffix KeyWord -LARGE_INTEGER -LDAP -LDAPMessage -LDAPURLDesc -LDAP_TIMEVAL -LINE -LLVMAttributeRef -LLVMBasicBlockRef -LLVMBuilderRef -LLVMContextRef -LLVMErrorRef -LLVMIntPredicate -LLVMJITEventListenerRef -LLVMJitContext -LLVMJitHandle -LLVMMemoryBufferRef -LLVMModuleRef -LLVMOrcCLookupSet -LLVMOrcCSymbolMapPair -LLVMOrcCSymbolMapPairs -LLVMOrcDefinitionGeneratorRef -LLVMOrcExecutionSessionRef -LLVMOrcJITDylibLookupFlags -LLVMOrcJITDylibRef -LLVMOrcJITTargetAddress -LLVMOrcJITTargetMachineBuilderRef -LLVMOrcLLJITBuilderRef -LLVMOrcLLJITRef -LLVMOrcLookupKind -LLVMOrcLookupStateRef -LLVMOrcMaterializationUnitRef -LLVMOrcObjectLayerRef -LLVMOrcResourceTrackerRef -LLVMOrcSymbolStringPoolRef -LLVMOrcThreadSafeContextRef -LLVMOrcThreadSafeModuleRef -LLVMPassBuilderOptionsRef -LLVMPassManagerBuilderRef -LLVMPassManagerRef -LLVMTargetMachineRef -LLVMTargetRef -LLVMTypeRef -LLVMValueRef -LOCALLOCK -LOCALLOCKOWNER -LOCALLOCKTAG -LOCALPREDICATELOCK -LOCK -LOCKMASK -LOCKMETHODID -LOCKMODE -LOCKTAG -LONG -LONG_PTR -LOOP -LPARAM -LPBYTE -LPCWSTR -LPSERVICE_STATUS -LPSTR -LPTHREAD_START_ROUTINE -LPTSTR -LPVOID -LPWSTR -LSEG -LUID -LVPagePruneState -LVRelState -LVSavedErrInfo -LWLock -LWLockHandle -LWLockMode -LWLockPadded -LWLockTrancheShmemData -LZ4F_compressionContext_t -LZ4F_decompressOptions_t -LZ4F_decompressionContext_t -LZ4F_errorCode_t -LZ4F_preferences_t -LZ4State LabelProvider LagTracker LargeObjectDesc Latch LauncherLastStartTimesEntry +lclContext +lclTocEntry +LDAP +LDAP_TIMEVAL +LDAPMessage +LDAPURLDesc +leaf_item +leafSegmentInfo LerpFunc LexDescr LexemeEntry @@ -1658,47 +1929,68 @@ LexemeHashKey LexemeInfo LexemeKey LexizeData +libpq_gettext_func +libpq_source +libpqsrv_PGresult LibraryInfo Limit LimitOption LimitPath LimitState LimitStateCond +LINE +line_t List +list_sort_comparator ListCell ListDictionary -ListParsedLex ListenAction ListenActionKind -ListenStmt ListenerEntry -LoInfo +ListenStmt +ListParsedLex LoadStmt +loc_chunk +local_relopt +local_relopts +local_source +local_ts_iter +local_ts_radix_tree LocalBufferLookupEnt +locale_t +LOCALLOCK +LOCALLOCKOWNER +LOCALLOCKTAG LocalPgBackendStatus +LOCALPREDICATELOCK LocalTransactionId -Location +locate_agg_of_level_context +locate_var_of_level_context +locate_windowfunc_context LocationIndex LocationLen +LOCK LockAcquireResult LockClauseStrength LockData LockInfoData +LockingClause LockInstanceData +LOCKMASK LockMethod LockMethodData +LOCKMETHODID +LOCKMODE LockRelId LockRows LockRowsPath LockRowsState LockStmt +LOCKTAG LockTagType LockTupleMode LockViewRecurse_context LockWaitPolicy -LockingClause -LogOpts -LogStmtLevel LogicalDecodeBeginCB LogicalDecodeBeginPrepareCB LogicalDecodeChangeCB @@ -1734,9 +2026,9 @@ LogicalRepCtxStruct LogicalRepMsgType LogicalRepPartMapEntry LogicalRepPreparedTxnData +LogicalRepRelation LogicalRepRelId LogicalRepRelMapEntry -LogicalRepRelation LogicalRepRollbackPreparedTxnData LogicalRepSequenceInfo LogicalRepStreamAbortData @@ -1749,37 +2041,75 @@ LogicalSlotInfo LogicalSlotInfoArr LogicalTape LogicalTapeSet -LookupSet +LogOpts +LogStmtLevel +logstreamer_param +LoInfo +LOOP +lquery +lquery_level +lquery_variant +LSEG LsnReadQueue LsnReadQueueNextFun LsnReadQueueNextStatus +ltree +ltree_gist +ltree_level LtreeGistOptions LtreeSignature +ltxtquery +LVPagePruneState +LVRelState +LVSavedErrInfo +LWLock +LWLockHandle +LWLockMode +LWLockPadded +LWLockTrancheShmemData +LZ4F_compressionContext_t +LZ4F_decompressionContext_t +LZ4F_decompressOptions_t +LZ4F_errorCode_t +LZ4F_preferences_t +LZ4State +mac8KEY +macaddr +macaddr_sortsupport_state +macaddr8 +macKEY MAGIC +manifest_data +manifest_file +manifest_files_hash +manifest_files_iterator +manifest_wal_range +manifest_writer +ManifestParseState +ManyTestResource +ManyTestResourceKind +map_variable_attnos_context +Material +MaterialPath +MaterialState +max_parallel_hazard_context +mb2wchar_with_len_converter +mbchar_verifier +mbcharacter_incrementer +mbdisplaylen_converter +mblen_converter +mbstr_verifier MBuf MCVHashContext MCVHashEntry MCVHashTable_hash MCVItem MCVList -MEMORY_BASIC_INFORMATION -MGVTBL -MINIDUMPWRITEDUMP -MINIDUMP_TYPE -MJEvalResult -MTTargetRelLookup -MVDependencies -MVDependency -MVNDistinct -MVNDistinctItem -ManyTestResource -ManyTestResourceKind -Material -MaterialPath -MaterialState -MdPathStr MdfdVec +MdPathStr Memoize +memoize_hash +memoize_iterator MemoizeEntry MemoizeInstrumentation MemoizeKey @@ -1811,24 +2141,38 @@ MergeStmt MergeSupportFunc MergeWhenClause MetaCommand +metastring +MGVTBL +MinimalTuple +MinimalTupleData +MinimalTupleTableSlot MinMaxAggInfo MinMaxAggPath MinMaxExpr +MinmaxMultiOpaque MinMaxMultiOptions MinMaxOp -MinimalTuple -MinimalTupleData -MinimalTupleTableSlot -MinmaxMultiOpaque MinmaxOpaque +missing_cache_key +mix_data_t +mixedStruct +MJEvalResult +mode_t ModifyTable ModifyTableContext ModifyTablePath ModifyTableState MonotonicFunction MorphOpaque +movedb_failure_params MsgType +MTTargetRelLookup MultiAssignRef +multirange_bsearch_comparison +multirange_unnest_fctx +MultirangeIOData +MultirangeParseState +MultirangeType MultiSortSupport MultiSortSupportData MultiXactId @@ -1838,23 +2182,16 @@ MultiXactOffset MultiXactOffset32 MultiXactStateData MultiXactStatus -MultirangeIOData -MultirangeParseState -MultirangeType -NDBOX -NDistinctParseState -NDistinctSemanticState -NLSVERSIONINFOEX -NODE -NTSTATUS -NUMCacheEntry -NUMDesc -NUMProc -NV +MVDependencies +MVDependency +MVNDistinct +MVNDistinctItem +mxact +mXactCacheEnt +mxtruncinfo Name -NameData -NameHashEntry NamedArgExpr +NameData NamedDSAState NamedDSHState NamedDSMState @@ -1862,53 +2199,63 @@ NamedLWLockTranche NamedLWLockTrancheRequest NamedTuplestoreScan NamedTuplestoreScanState +NameHashEntry NamespaceInfo +NDBOX +NDistinctParseState +NDistinctSemanticState +needs_fmgr_hook_type NestLoop NestLoopParam NestLoopState NestPath +network_sortsupport_state NewColumnValue NewConstraint NextSampleBlock_function NextSampleTuple_function NextValueExpr +nl_item Node +NODE NodeInstrumentation NodeInterfaceTuple +nodeitem NodeTag NodeTuple -NonEmptyRange NoneCompressorState -NotNullSource +NonEmptyRange +normal_rand_fctx Notification NotificationHash NotificationList NotifyStmt NotnullHashEntry +NotNullSource +nsphash_hash Nsrt -NtDllRoutine -NtFlushBuffersFileEx_t +ntile_context +NullableDatum NullIfExpr +nullingrel_info +NullingRelsMatch NullTest NullTestType -NullableDatum -NullingRelsMatch +NUMCacheEntry +NUMDesc +numeric Numeric NumericAggState NumericDigit NumericSortSupport NumericSumAccum NumericVar +NUMProc +NV OAuthValidatorCallbacks OAuthValidatorModuleInit -OM_uint32 -OP -OSAPerGroupState -OSAPerQueryState -OSInfo -OSSLCipher -OSSLDigest -OVERLAPPED +object_access_hook_type +object_access_hook_type_str ObjectAccessDrop ObjectAccessNamespaceSearch ObjectAccessPostAlter @@ -1916,25 +2263,31 @@ ObjectAccessPostCreate ObjectAccessType ObjectAddress ObjectAddressAndFlags +ObjectAddresses ObjectAddressExtra ObjectAddressStack -ObjectAddresses ObjectClass ObjectPropertyType ObjectType ObjectWithArgs +off_t Offset OffsetNumber OffsetVarNodes_context Oid Oid8 +oidKEY OidOptions +oidvector OkeysState OldMultiXactReader OldSnapshotControlData OldSnapshotTimeMapping OldToNewMapping OldToNewMappingData +OM_uint32 +on_dsm_detach_callback +on_exit_nicely_callback OnCommitAction OnCommitItem OnConflictAction @@ -1942,257 +2295,56 @@ OnConflictActionState OnConflictClause OnConflictExpr OnConflictSetState +OP OpBtreeInterpretation OpClassCacheEnt -OpExpr -OpFamilyMember -OpFamilyOpFuncGroup -OpIndexInterpretation OpclassInfo +openssl_tls_init_hook_typ Operator OperatorElement +OpExpr OpfamilyInfo +OpFamilyMember +OpFamilyOpFuncGroup +OpIndexInterpretation OprCacheEntry OprCacheKey OprInfo OprProofCacheEntry OprProofCacheKey OrArgIndexMatch +OSAPerGroupState +OSAPerQueryState +OSInfo +ossl_EVP_cipher_func +OSSLCipher +OSSLDigest OuterJoinClauseInfo +output_type OutputPluginCallbacks OutputPluginOptions OutputPluginOutputType OutputPluginParam OutputPluginParamKey +overexplain_options OverrideSearchPath OverrideStackEntry OverridingKind -PACE_HEADER -PACL -PATH -PCtxtHandle -PERL_CONTEXT -PERL_SI -PFN -PGAlignedBlock -PGAlignedXLogBlock -PGAsyncStatusType -PGCALL2 -PGCRYPTO_SHA_t -PGChecksummablePage -PGContextVisibility -PGEvent -PGEventConnDestroy -PGEventConnReset -PGEventId -PGEventProc -PGEventRegister -PGEventResultCopy -PGEventResultCreate -PGEventResultDestroy -PGFInfoFunction -PGFileType -PGFunction -PGIOAlignedBlock -PGLZ_HistEntry -PGLZ_Strategy -PGLoadBalanceType -PGMessageField -PGModuleMagicFunction -PGNoticeHooks -PGOutputData -PGOutputTxnData -PGPROC -PGP_CFB -PGP_Context -PGP_MPI -PGP_PubKey -PGP_S2K -PGPing -PGQueryClass -PGRUsage -PGSemaphore -PGSemaphoreData -PGShmemHeader -PGTargetServerType -PGTernaryBool -PGTransactionStatusType -PGVerbosity -PG_Locale_Strategy -PG_Lock_Status -PG_init_t -PGauthData -PGcancel -PGcancelConn -PGcmdQueueEntry -PGconn -PGdataValue -PGlobjfuncs -PGnotify -PGoauthBearerRequest -PGoauthBearerRequestV2 -PGpipelineStatus -PGpromptOAuthDevice -PGresAttDesc -PGresAttValue -PGresParamDesc -PGresult -PGresult_data -PIO_STATUS_BLOCK -PLAINTREE -PLAssignStmt -PLcword -PLpgSQL_case_when -PLpgSQL_condition -PLpgSQL_datum -PLpgSQL_datum_type -PLpgSQL_diag_item -PLpgSQL_exception -PLpgSQL_exception_block -PLpgSQL_execstate -PLpgSQL_expr -PLpgSQL_func_hashkey -PLpgSQL_function -PLpgSQL_getdiag_kind -PLpgSQL_if_elsif -PLpgSQL_label_type -PLpgSQL_nsitem -PLpgSQL_nsitem_type -PLpgSQL_plugin -PLpgSQL_promise_type -PLpgSQL_raise_option -PLpgSQL_raise_option_type -PLpgSQL_rec -PLpgSQL_recfield -PLpgSQL_resolve_option -PLpgSQL_row -PLpgSQL_rwopt -PLpgSQL_stmt -PLpgSQL_stmt_assert -PLpgSQL_stmt_assign -PLpgSQL_stmt_block -PLpgSQL_stmt_call -PLpgSQL_stmt_case -PLpgSQL_stmt_close -PLpgSQL_stmt_commit -PLpgSQL_stmt_dynexecute -PLpgSQL_stmt_dynfors -PLpgSQL_stmt_execsql -PLpgSQL_stmt_exit -PLpgSQL_stmt_fetch -PLpgSQL_stmt_forc -PLpgSQL_stmt_foreach_a -PLpgSQL_stmt_fori -PLpgSQL_stmt_forq -PLpgSQL_stmt_fors -PLpgSQL_stmt_getdiag -PLpgSQL_stmt_if -PLpgSQL_stmt_loop -PLpgSQL_stmt_open -PLpgSQL_stmt_perform -PLpgSQL_stmt_raise -PLpgSQL_stmt_return -PLpgSQL_stmt_return_next -PLpgSQL_stmt_return_query -PLpgSQL_stmt_rollback -PLpgSQL_stmt_type -PLpgSQL_stmt_while -PLpgSQL_trigtype -PLpgSQL_type -PLpgSQL_type_type -PLpgSQL_var -PLpgSQL_variable -PLwdatum -PLword -PLyArrayToOb -PLyCursorObject -PLyDatumToOb -PLyDatumToObFunc -PLyExceptionEntry -PLyExecutionContext -PLyObToArray -PLyObToDatum -PLyObToDatumFunc -PLyObToDomain -PLyObToScalar -PLyObToTransform -PLyObToTuple -PLyObject_AsString_t -PLyPlanObject -PLyProcedure -PLyProcedureCache -PLyProcedureEntry -PLyProcedureKey -PLyResultObject -PLySRFState -PLySavedArgs -PLyScalarToOb -PLySubtransactionData -PLySubtransactionObject -PLyTransformToOb -PLyTrigType -PLyTupleToOb -PLyUnicode_FromStringAndSize_t -PLy_elog_impl_t -PMChild -PMChildPool -PMINIDUMP_CALLBACK_INFORMATION -PMINIDUMP_EXCEPTION_INFORMATION -PMINIDUMP_USER_STREAM_INFORMATION -PMSignalData -PMSignalReason -PMState -POLYGON -PQArgBlock -PQEnvironmentOption -PQExpBuffer -PQExpBufferData -PQauthDataHook_type -PQcommMethods -PQconninfoOption -PQnoticeProcessor -PQnoticeReceiver -PQprintOpt -PQsslKeyPassHook_OpenSSL_type -PREDICATELOCK -PREDICATELOCKTAG -PREDICATELOCKTARGET -PREDICATELOCKTARGETTAG -PROCESS_INFORMATION -PROCLOCK -PROCLOCKTAG -PROC_HDR -PSID -PSQL_COMP_CASE -PSQL_ECHO -PSQL_ECHO_HIDDEN -PSQL_ERROR_ROLLBACK -PSQL_SEND_MODE -PTEntryArray -PTIterationArray -PTOKEN_PRIVILEGES -PTOKEN_USER -PUTENVPROC -PVIndStats -PVIndVacStatus -PVOID -PVShared -PVSharedCostParams -PVWorkerStats -PVWorkerUsage -PX_Alias -PX_Cipher -PX_Combo -PX_HMAC -PX_MD Page PageData PageGistNSN PageHeader PageHeaderData -PageXLogRecPtr +pagetable_hash +pagetable_iterator PagetableEntry +PageXLogRecPtr +pairingheap +pairingheap_comparator +pairingheap_node Pairs +pam_handle_t +parallel_worker_main_type ParallelAppendState ParallelApplyWorkerEntry ParallelApplyWorkerInfo @@ -2231,23 +2383,26 @@ ParamPathInfo ParamRef ParamsErrorCbData ParentMapEntry +parse_error_callback_arg ParseCallbackState +ParsedLex +ParsedScript +ParsedText +ParsedWord ParseExprKind ParseLoc ParseNamespaceColumn ParseNamespaceItem ParseParamRefHook -ParseState -ParsedLex -ParsedScript -ParsedText -ParsedWord +parser_context ParserSetupHook ParserState +ParseState PartClauseInfo PartClauseMatchStatus PartClauseTarget PartialFileSetState +partition_method_t PartitionBoundInfo PartitionBoundInfoData PartitionBoundSpec @@ -2258,6 +2413,8 @@ PartitionDirectory PartitionDirectoryData PartitionDirectoryEntry PartitionDispatch +PartitionedRelPruneInfo +PartitionedRelPruningData PartitionElem PartitionHashBound PartitionKey @@ -2279,45 +2436,120 @@ PartitionSchemeData PartitionSpec PartitionStrategy PartitionTupleRouting -PartitionedRelPruneInfo -PartitionedRelPruningData PartitionwiseAggregateType PasswordType Path +PATH PathClauseUsage PathCostComparison PathHashStack PathKey PathKeysComparison PathTarget -PatternInfo -PatternInfoArray Pattern_Prefix_Status Pattern_Type +PatternInfo +PatternInfoArray +PeerNodeInfo +pending_label PendingFsyncEntry PendingListenAction PendingListenEntry +pendingPosition PendingRelDelete PendingRelSync PendingUnlinkEntry PendingWrite PendingWriteback -PerLockTagEntry -PerlInterpreter +PERL_CONTEXT Perl_ppaddr_t -Permutation -PermutationStep -PermutationStepBlocker -PermutationStepBlockerType +PERL_SI +PerlInterpreter +PerLockTagEntry +PFN +Pg_abi_values +pg_atomic_flag +pg_atomic_uint32 +pg_atomic_uint64 +pg_be_sasl_mech +pg_case_map +pg_category_range +pg_checksum_context +pg_checksum_raw_context +pg_checksum_type +pg_compress_algorithm +pg_compress_specification +pg_conn_host +pg_conn_host_type +pg_conv_map +pg_crc32 +pg_crc32c +pg_cryptohash_ctx +pg_cryptohash_errno +pg_cryptohash_type +pg_ctype_cache +pg_enc +pg_enc2gettext +pg_enc2name +pg_encname +pg_fe_sasl_mech +Pg_finfo_record +pg_funcptr_t +pg_getopt_ctx +pg_gssinfo +pg_hmac_ctx +pg_hmac_errno +PG_init_t +pg_int64 +pg_local_to_utf_combined +PG_Locale_Strategy +pg_locale_t +PG_Lock_Status +Pg_magic_struct +pg_mb_radix_tree +pg_on_exit_callback +pg_plan_advice_advisor_hook +pg_prng_state +pg_re_flags +pg_regex_t +pg_regmatch_t +pg_regoff_t +pg_saslprep_rc +pg_saslprep_test_context +pg_signal_info +pg_snapshot +pg_special_case +pg_stack_base_t +pg_ternary +pg_time_t +pg_time_usec_t +pg_tz +pg_tz_cache +pg_tzenum +pg_unicode_category +pg_unicode_decompinfo +pg_unicode_decomposition +pg_unicode_norminfo +pg_unicode_normprops +pg_unicode_properties +pg_unicode_range +pg_unicode_recompinfo +pg_usec_time_t +pg_utf_to_local_combined +pg_utf8_codepoint_range +pg_uuid_t +pg_wc_probefunc +pg_wchar +pg_wchar_tbl PgAioBackend PgAioCtl PgAioHandle PgAioHandleCallbackComplete PgAioHandleCallbackID PgAioHandleCallbackReport -PgAioHandleCallbackStage PgAioHandleCallbacks PgAioHandleCallbacksEntry +PgAioHandleCallbackStage PgAioHandleFlags PgAioHandleState PgAioOp @@ -2335,7 +2567,11 @@ PgAioWorkerControl PgAioWorkerSet PgAioWorkerSlot PgAioWorkerSubmissionQueue +PGAlignedBlock +PGAlignedXLogBlock PgArchData +PGAsyncStatusType +PGauthData PgBackendGSSStatus PgBackendSSLStatus PgBackendStatus @@ -2346,7 +2582,25 @@ PgBenchExprType PgBenchFunction PgBenchValue PgBenchValueType +PGCALL2 +PGcancel +PGcancelConn +PGChecksummablePage PgChecksumMode +PGcmdQueueEntry +PGconn +PGContextVisibility +PGCRYPTO_SHA_t +PGdataValue +PGEvent +PGEventConnDestroy +PGEventConnReset +PGEventId +PGEventProc +PGEventRegister +PGEventResultCopy +PGEventResultCreate +PGEventResultDestroy PgFdwAnalyzeState PgFdwConnState PgFdwDirectModifyState @@ -2356,26 +2610,95 @@ PgFdwPathExtraData PgFdwRelationInfo PgFdwSamplingMethod PgFdwScanState +PGFileType +PGFInfoFunction +PGFunction PgIfAddrCallback -PgStatShared_Archiver -PgStatShared_Backend -PgStatShared_BgWriter -PgStatShared_Checkpointer -PgStatShared_Common -PgStatShared_CustomFixedEntry -PgStatShared_CustomVarEntry -PgStatShared_Database -PgStatShared_Function -PgStatShared_HashEntry -PgStatShared_IO -PgStatShared_InjectionPoint -PgStatShared_InjectionPointFixed -PgStatShared_Lock -PgStatShared_Relation -PgStatShared_ReplSlot -PgStatShared_SLRU -PgStatShared_Subscription -PgStatShared_Wal +PGIOAlignedBlock +PGLoadBalanceType +PGlobjfuncs +PGLZ_HistEntry +PGLZ_Strategy +PGMessageField +PGModuleMagicFunction +PGNoticeHooks +PGnotify +PGoauthBearerRequest +PGoauthBearerRequestV2 +pgoff_t +PGOutputData +PGOutputTxnData +pgp_armor_headers_state +PGP_CFB +PGP_Context +PGP_MPI +PGP_PubKey +PGP_S2K +pgpa_advice_item +pgpa_advice_tag_type +pgpa_advice_target +pgpa_identifier +pgpa_index_target +pgpa_itm_type +pgpa_jo_outcome +pgpa_join_member +pgpa_join_state +pgpa_join_strategy +pgpa_join_unroller +pgpa_output_context +pgpa_plan_walker_context +pgpa_planner_info +pgpa_planner_state +pgpa_qf_type +pgpa_query_feature +pgpa_scan +pgpa_scan_strategy +pgpa_target_type +pgpa_trove +pgpa_trove_entry +pgpa_trove_entry_element +pgpa_trove_entry_hash +pgpa_trove_entry_key +pgpa_trove_lookup_type +pgpa_trove_result +pgpa_trove_slice +pgpa_unrolled_join +pgpa_yy_extra_type +pgParameterStatus +PGPing +PGpipelineStatus +PGPROC +PGpromptOAuthDevice +PGQueryClass +PGresAttDesc +PGresAttValue +PGresParamDesc +PGresult +PGresult_data +PGRUsage +pgsa_entry +pgsa_entry_key +pgsa_saved_entry +pgsa_saved_stash +pgsa_saved_stash_table_hash +pgsa_saved_stash_table_iterator +pgsa_shared_state +pgsa_stash +pgsa_stash_count +pgsa_stash_count_table_hash +pgsa_stash_name +pgsa_stash_name_table_hash +pgsa_writer_context +PGSemaphore +PGShmemHeader +pgsocket +pgsql_thing_t +pgssEntry +pgssGlobalStats +pgssHashKey +pgssSharedState +pgssStoreKind +pgssVersion PgStat_ArchiverStats PgStat_Backend PgStat_BackendPending @@ -2384,6 +2707,8 @@ PgStat_BgWriterStats PgStat_BktypeIO PgStat_CheckpointerStats PgStat_Counter +pgstat_entry_ref_hash_hash +pgstat_entry_ref_hash_iterator PgStat_EntryRef PgStat_EntryRefHashEntry PgStat_FetchConsistency @@ -2396,13 +2721,15 @@ PgStat_KindInfo PgStat_LocalState PgStat_Lock PgStat_LockEntry +pgstat_page PgStat_PendingDroppedStatsItem PgStat_PendingIO PgStat_PendingLock PgStat_PendingWalStats -PgStat_SLRUStats PgStat_ShmemControl +PgStat_SLRUStats PgStat_Snapshot +pgstat_snapshot_hash PgStat_SnapshotEntry PgStat_StatCustomFixedEntry PgStat_StatCustomVarEntry @@ -2411,38 +2738,188 @@ PgStat_StatFuncEntry PgStat_StatInjEntry PgStat_StatInjFixedEntry PgStat_StatReplSlotEntry +PgStat_StatsFileOp PgStat_StatSubEntry PgStat_StatTabEntry -PgStat_StatsFileOp PgStat_SubXactStatus PgStat_TableCounts PgStat_TableStatus PgStat_TableXactStatus PgStat_WalCounters PgStat_WalStats -PgXmlErrorContext -PgXmlStrictness -Pg_abi_values -Pg_finfo_record -Pg_magic_struct +PgStatShared_Archiver +PgStatShared_Backend +PgStatShared_BgWriter +PgStatShared_Checkpointer +PgStatShared_Common +PgStatShared_CustomFixedEntry +PgStatShared_CustomVarEntry +PgStatShared_Database +PgStatShared_Function +PgStatShared_HashEntry +PgStatShared_InjectionPoint +PgStatShared_InjectionPointFixed +PgStatShared_IO +PgStatShared_Lock +PgStatShared_Relation +PgStatShared_ReplSlot +PgStatShared_SLRU +PgStatShared_Subscription +PgStatShared_Wal +pgstattuple_type +PGTargetServerType +PGTernaryBool +pgthreadlock_t +PGTransactionStatusType +PGVerbosity +PgXmlErrorContext +PgXmlStrictness +pid_t PipeProtoChunk PipeProtoHeader +pivot_field PlaceHolderInfo PlaceHolderVar +PLAINTREE Plan PlanDirectModify_function PlanForeignModify_function PlanInvalItem -PlanRowMark -PlanState PlannedStmt PlannedStmtOrigin +planner_hook_type +planner_setup_hook_type +planner_shutdown_hook_type PlannerGlobal PlannerInfo PlannerParamItem +PlanRowMark +PlanState +planstate_tree_walker_callback +PLAssignStmt +PLcword +plperl_array_info +plperl_call_data +plperl_interp_desc +plperl_proc_desc +plperl_proc_key +plperl_proc_ptr +plperl_query_desc +plperl_query_entry +PLpgSQL_case_when +plpgsql_CastExprHashEntry +plpgsql_CastHashEntry +plpgsql_CastHashKey +PLpgSQL_condition +PLpgSQL_datum +PLpgSQL_datum_type +PLpgSQL_diag_item +PLpgSQL_exception +PLpgSQL_exception_block +PLpgSQL_execstate +PLpgSQL_expr +plpgsql_expr_walker_callback +PLpgSQL_func_hashkey +PLpgSQL_function +PLpgSQL_getdiag_kind +plpgsql_HashEnt +PLpgSQL_if_elsif +PLpgSQL_label_type +PLpgSQL_nsitem +PLpgSQL_nsitem_type +PLpgSQL_plugin +PLpgSQL_promise_type +PLpgSQL_raise_option +PLpgSQL_raise_option_type +PLpgSQL_rec +PLpgSQL_recfield +PLpgSQL_resolve_option +PLpgSQL_row +PLpgSQL_rwopt +PLpgSQL_stmt +PLpgSQL_stmt_assert +PLpgSQL_stmt_assign +PLpgSQL_stmt_block +PLpgSQL_stmt_call +PLpgSQL_stmt_case +PLpgSQL_stmt_close +PLpgSQL_stmt_commit +PLpgSQL_stmt_dynexecute +PLpgSQL_stmt_dynfors +PLpgSQL_stmt_execsql +PLpgSQL_stmt_exit +PLpgSQL_stmt_fetch +PLpgSQL_stmt_forc +PLpgSQL_stmt_foreach_a +PLpgSQL_stmt_fori +PLpgSQL_stmt_forq +PLpgSQL_stmt_fors +PLpgSQL_stmt_getdiag +PLpgSQL_stmt_if +PLpgSQL_stmt_loop +PLpgSQL_stmt_open +PLpgSQL_stmt_perform +PLpgSQL_stmt_raise +PLpgSQL_stmt_return +PLpgSQL_stmt_return_next +PLpgSQL_stmt_return_query +PLpgSQL_stmt_rollback +PLpgSQL_stmt_type +plpgsql_stmt_walker_callback +PLpgSQL_stmt_while +PLpgSQL_trigtype +PLpgSQL_type +PLpgSQL_type_type +PLpgSQL_var +PLpgSQL_variable +pltcl_call_state +pltcl_interp_desc +pltcl_proc_desc +pltcl_proc_key +pltcl_proc_ptr +pltcl_query_desc +PLwdatum +PLword +PLy_elog_impl_t +PLyArrayToOb +PLyCursorObject +PLyDatumToOb +PLyDatumToObFunc +PLyExceptionEntry +PLyExecutionContext +PLyObject_AsString_t +PLyObToArray +PLyObToDatum +PLyObToDatumFunc +PLyObToDomain +PLyObToScalar +PLyObToTransform +PLyObToTuple +PLyPlanObject +PLyProcedure +PLyProcedureCache +PLyProcedureEntry +PLyProcedureKey +PLyResultObject +PLySavedArgs +PLyScalarToOb +PLySRFState +PLySubtransactionData +PLySubtransactionObject +PLyTransformToOb +PLyTrigType +PLyTupleToOb +PLyUnicode_FromStringAndSize_t +PMChild +PMChildPool +PMSignalData +PMSignalReason +PMState Point Pointer PolicyInfo +POLYGON +polymorphic_actuals PolyNumAggState Pool PopulateArrayContext @@ -2454,34 +2931,75 @@ Portal PortalHashEnt PortalStatus PortalStrategy -PostParseColumnRefHook -PostRewriteHook +pos_trgm +post_parse_analyze_hook_type PostgresPollingStatusType PostingItem -PreParseColumnRefHook +PostParseColumnRefHook +PostRewriteHook +PQArgBlock +PQauthDataHook_type +pqbool +PQcommMethods +PQconninfoOption +PQEnvironmentOption +PQExpBuffer +PQExpBufferData +PQnoticeProcessor +PQnoticeReceiver +PQprintOpt +pqsigfunc +pqsigfunc_legacy +PQsslKeyPassHook_OpenSSL_type PredClass +PREDICATELOCK +PredicateLockData +PREDICATELOCKTAG +PREDICATELOCKTARGET +PREDICATELOCKTARGETTAG +PredicateLockTargetType PredIterInfo PredIterInfoData PredXactList -PredicateLockData -PredicateLockTargetType PrefetchBufferResult -PrepParallelRestorePtrType -PrepareStmt PreparedStatement +PrepareStmt +PreParseColumnRefHook +PrepParallelRestorePtrType PresortedKeyData PrewarmType PrintExtraTocPtrType -PrintTocDataPtrType PrintfArgType PrintfArgValue PrintfTarget +printQueryOpt +printTableContent +printTableFooter +printTableOpt +printTextFormat +printTextLineFormat +printTextLineWrap +printTextRule +PrintTocDataPtrType PrinttupAttrInfo -PrivTarget +printXheaderWidthType +priv_map PrivateRefCountData PrivateRefCountEntry +PrivTarget +PROC_HDR ProcArrayStruct +process_file_callback_t +process_sublinks_context +ProcessingMode +ProcessUtility_hook_type +ProcessUtilityContext ProcLangInfo +proclist_head +proclist_mutable_iter +proclist_node +PROCLOCK +PROCLOCKTAG ProcNumber ProcSignalBarrierType ProcSignalHeader @@ -2489,16 +3007,14 @@ ProcSignalReason ProcSignalSlot ProcState ProcWaitStatus -ProcessUtilityContext -ProcessUtility_hook_type -ProcessingMode ProgressCommandType +ProjectionInfo +ProjectionPath ProjectSet ProjectSetPath ProjectSetState -ProjectionInfo -ProjectionPath PromptInterruptContext +promptStatus_t PropGraphEdge PropGraphLabelAndProperties PropGraphProperties @@ -2510,6 +3026,11 @@ PruneFreezeResult PruneReason PruneState PruneStepResult +PSQL_COMP_CASE +PSQL_ECHO +PSQL_ECHO_HIDDEN +PSQL_ERROR_ROLLBACK +PSQL_SEND_MODE PsqlScanCallbacks PsqlScanQuoteType PsqlScanResult @@ -2517,7 +3038,16 @@ PsqlScanState PsqlScanStateData PsqlScanStateSave PsqlSettings +PTEntryArray +pthread_barrier_t +pthread_key_t +pthread_mutex_t +pthread_once_t +pthread_t +PTIterationArray +ptrdiff_t Publication +publication_tables_state PublicationActions PublicationAllObjSpec PublicationAllObjType @@ -2530,29 +3060,51 @@ PublicationRelInfo PublicationRelKind PublicationSchemaInfo PublicationTable +published_rel PublishGencolsType +pull_var_clause_context +pull_varattnos_context +pull_varnos_context +pull_vars_context PullFilter PullFilterOps +pullup_replace_vars_context +pushdown_safe_type +pushdown_safety_info PushFilter PushFilterOps PushFunction -PyCFunction -PyMappingMethods -PyMethodDef -PyModuleDef -PyObject -PySequenceMethods -PyTypeObject +PVIndStats +PVIndVacStatus +PVShared +PVSharedCostParams +PVWorkerStats +PVWorkerUsage +PX_Alias +PX_Cipher +PX_Combo +PX_HMAC +PX_MD +Py_ssize_t +PyCFunction +PyMappingMethods +PyMethodDef +PyModuleDef +PyObject +PySequenceMethods PyType_Slot PyType_Spec -Py_ssize_t +PyTypeObject +qc_hash_func QPRS_STATE +qsort_arg_comparator +qsort_comparator QTN2QTState QTNode -QUERYTYPE QualCost QualItem Query +query_pathkeys_callback QueryCompletion QueryDesc QueryEnvironment @@ -2565,29 +3117,14 @@ QueryOperator QueryRepresentation QueryRepresentationOperand QuerySource +QUERYTYPE QueueBackendStatus +QueuedMessage QueuePosition QueueTuple -QueuedMessage QuitSignalReason -RBTNode -RBTOrderControl -RBTree -RBTreeIterator -REPARSE_JUNCTION_DATA_BUFFER -RIX -RI_CompareHashEntry -RI_CompareKey -RI_ConstraintInfo -RI_FastPathEntry -RI_FastPathKey -RI_QueryHashEntry -RI_QueryKey -RTEKind -RTEPermissionInfo -RWConflict -RWConflictData -RWConflictPoolHeader +radius_attribute +radius_packet RadixSortInfo Range RangeBound @@ -2596,7 +3133,9 @@ RangeFunction RangeGraphTable RangeIOData RangeQueryClause +Ranges RangeSubselect +rangeTableEntry_used_context RangeTableFunc RangeTableFuncCol RangeTableSample @@ -2606,15 +3145,21 @@ RangeTblRef RangeType RangeVar RangeVarGetRelidCallback -Ranges +rank_context RawColumnDefault RawParseMode RawStmt -ReInitializeDSMForeignScan_function -ReScanForeignScan_function -ReadBufPtrType +rbt_allocfunc +rbt_combiner +rbt_comparator +rbt_freefunc +RBTNode +RBTOrderControl +RBTree +RBTreeIterator ReadBufferMode ReadBuffersOperation +ReadBufPtrType ReadBytePtrType ReadExtraTocPtrType ReadFunc @@ -2641,55 +3186,85 @@ RecursionContext RecursiveUnion RecursiveUnionPath RecursiveUnionState +reduce_outer_joins_partial_state +reduce_outer_joins_pass1_state +reduce_outer_joins_pass2_state +refcount_hash +refcount_iterator RefetchForeignRow_function RefreshMatViewStmt -RegProcedure +regc_wc_probefunc +regex_arc_t +regexp_matches_ctx Regis RegisNode +registered_buffer RegisteredBgWorker +regproc +RegProcedure ReindexErrorInfo ReindexIndexInfo ReindexObjectType ReindexParams ReindexStmt ReindexType +ReInitializeDSMForeignScan_function +RelabelType RelAggInfo +Relation +RelationData +RelationInfo +RelationPtr +RelationSyncEntry +RelcacheCallbackFunction +ReleaseMatchCB RelFileLocator RelFileLocatorBackend RelFileNumber +RelfilenumberMapEntry +RelfilenumberMapKey RelIdCacheEnt +Relids RelIdToTypeIdCacheEntry RelInfo RelInfoArr RelMapFile RelMapping +RelocationBufferInfo +relopt_bool +relopt_enum +relopt_enum_elt_def +relopt_gen +relopt_int +relopt_kind +relopt_parse_elt +relopt_real +relopt_string +relopt_ternary +relopt_type +relopt_value RelOptInfo RelOptKind +relopts_validator RelPathStr +RelptrFreePageBtree +RelptrFreePageManager +RelptrFreePageSpanLeader RelStatsInfo RelSyncCallbackFunction RelToCheck RelToCluster -RelabelType -Relation -RelationData -RelationInfo -RelationPtr -RelationSyncEntry -RelcacheCallbackFunction -ReleaseMatchCB -RelfilenumberMapEntry -RelfilenumberMapKey -Relids -RelocationBufferInfo -RelptrFreePageBtree -RelptrFreePageManager -RelptrFreePageSpanLeader RemoteAttributeMapping +remoteConn +remoteConnHashEnt +remoteDep +RemoteInfo RemoteSlot RemoteStatsResults RemoteSyncPosition +remove_nulling_relids_context RenameStmt +rendezvousHashEntry ReopenPtrType ReorderBuffer ReorderBufferApplyChangeCB @@ -2713,24 +3288,20 @@ ReorderBufferStreamPrepareCB ReorderBufferStreamStartCB ReorderBufferStreamStopCB ReorderBufferStreamTruncateCB -ReorderBufferTXN -ReorderBufferTXNByIdEnt ReorderBufferToastEnt ReorderBufferTupleBuf ReorderBufferTupleCidEnt ReorderBufferTupleCidKey +ReorderBufferTXN +ReorderBufferTXNByIdEnt ReorderBufferUpdateProgressTxnCB ReorderTuple -RepOriginId -RepSetSeqTuple -RepSetTableTuple -RepSetTuple RepackCommand RepackDecodingState RepackStmt ReparameterizeForeignPathByChild_function -ReplOriginId -ReplOriginXactState +replace_rte_variables_callback +replace_rte_variables_context ReplaceVarsFromTargetList_context ReplaceVarsNoMatchOption ReplaceWrapOption @@ -2745,7 +3316,17 @@ ReplicationSlotPersistentData ReplicationState ReplicationStateCtl ReplicationStateOnDisk -ResTarget +ReplOriginId +ReplOriginXactState +RepOriginId +report_error_fn +RepsetCapture +RepsetFingerprintEntry +RepSetSeqTuple +RepsetTableCapture +RepSetTableTuple +RepSetTuple +ReScanForeignScan_function ReservedObjectKind ReservedObjectPurpose ReservoirState @@ -2758,6 +3339,7 @@ ResourceReleaseCallback ResourceReleaseCallbackItem ResourceReleasePhase ResourceReleasePriority +ResTarget RestoreOptions RestorePass RestrictInfo @@ -2769,28 +3351,45 @@ RetainDeadTuplesData RetainDeadTuplesPhase RetrievedObject RetrievedObjects -ReturnSetInfo -ReturnStmt ReturningClause ReturningExpr ReturningOption ReturningOptionKind +ReturnSetInfo +ReturnStmt +ReverseSubTarget RevmapContents RevokeRoleGrantAction +rewind_source +rewrite_event RewriteMappingDataEntry RewriteMappingFile RewriteRule RewriteState RewriteStateData +rf_context +rfile +RI_CompareHashEntry +RI_CompareKey +RI_ConstraintInfo +RI_FastPathEntry +RI_FastPathKey +RI_QueryHashEntry +RI_QueryKey +RIX +rm_detail_t RmgrData RmgrDescData RmgrId +role_auth_extra +rolename_hash RoleNameEntry RoleNameItem RoleSpec RoleSpecType RoleStmtType RollupData +row_security_policy_hook_type RowCompareExpr RowCompareType RowExpr @@ -2799,68 +3398,28 @@ RowMarkClause RowMarkType RowSecurityDesc RowSecurityPolicy -RtlGetLastNtStatus_t -RtlNtStatusToDosError_t +rsv_callback +rt_iter +rt_node_class_test_elem +rt_radix_tree +RTEKind +RTEPermissionInfo RuleInfo RuleLock RuleStmt RunMode RunningTransactions RunningTransactionsData -SASLStatus -SC_HANDLE -SECURITY_ATTRIBUTES -SECURITY_STATUS -SEG -SERIALIZABLEXACT -SERIALIZABLEXID -SERIALIZABLEXIDTAG -SERVICE_STATUS -SERVICE_STATUS_HANDLE -SERVICE_TABLE_ENTRY -SID_AND_ATTRIBUTES -SID_IDENTIFIER_AUTHORITY -SID_NAME_USE -SISeg -SIZE_T -SMgrRelation -SMgrRelationData -SMgrSortArray -SN_local -SOCKADDR -SOCKET -SPELL -SPICallbackArg -SPIExecuteOptions -SPIParseOpenOptions -SPIPlanPtr -SPIPrepareOptions -SPITupleTable -SPKFlushPosition -SPKRelMetaCacheEntry -SPLITCOST -SPNode -SPNodeData -SPPageDesc -SQLDropObject -SQLFunctionCache -SQLFunctionCachePtr -SQLFunctionHashEntry -SQLFunctionParseInfo -SQLFunctionParseInfoPtr -SQLValueFunction -SQLValueFunctionOp -SSL -SSLExtensionInfoContext -SSL_CTX -STARTUPINFO -STRLEN -SV -SYNCHRONIZATION_BARRIER -SYSTEM_INFO +RWConflict +RWConflictData +RWConflictPoolHeader SampleScan SampleScanGetSampleSize_function SampleScanState +saophash_hash +SASLStatus +save_buffer +save_locale_t SavedTransactionCharacteristics ScalarArrayOpExpr ScalarArrayOpExprHashEntry @@ -2874,25 +3433,30 @@ ScanKey ScanKeyData ScanKeywordHashFunc ScanKeywordList +ScannerCallbackState ScanState ScanTypeControl -ScannerCallbackState SchemaQuery +scram_state +scram_state_enum +script_error_callback_arg SearchPathCacheEntry SearchPathCacheKey SearchPathMatcher -SecBuffer -SecBufferDesc SecLabelItem SecLabelStmt +security_class_t SeenRelsEntry +SEG +Selectivity SelectLimit SelectStmt SelectStmtPassthrough -Selectivity SelfJoinCandidate -SemTPadded +sem_t SemiAntiJoinFactors +SemTPadded +sepgsql_context_info_t SeqScan SeqScanInstrumentation SeqScanState @@ -2900,22 +3464,31 @@ SeqStateTuple SeqTable SeqTableData SeqType +sequence_magic +SequenceCapture SequenceItem SerCommitSeqNo SerialControl SerialIOData +SERIALIZABLEXACT SerializableXactHandle -SerializeDestReceiver -SerializeMetrics +SERIALIZABLEXID +SERIALIZABLEXIDTAG SerializedActiveRelMaps SerializedClientConnectionInfo +SerializeDestReceiver SerializedRanges SerializedReindexState SerializedSnapshotData SerializedTransactionState +SerializeMetrics Session SessionBackupState SessionEndType +set_conn_altsock_func +set_conn_oauth_token_func +set_join_pathlist_hook_type +set_rel_pathlist_hook_type SetConstraintState SetConstraintStateData SetConstraintTriggerData @@ -2924,19 +3497,20 @@ SetFunctionReturnMode SetHintBitsState SetOp SetOpCmd +SetOperation +SetOperationStmt SetOpPath SetOpState SetOpStatePerGroup SetOpStatePerGroupData SetOpStatePerInput SetOpStrategy -SetOperation -SetOperationStmt SetQuantifier SetToDefault -SetVarReturningType_context SetupWorkerPtrType -ShDependObjectInfo +SetVarReturningType_context +shared_ts_iter +shared_ts_radix_tree SharedAggInfo SharedBitmapHeapInstrumentation SharedBitmapState @@ -2949,18 +3523,19 @@ SharedIncrementalSortInfo SharedIndexScanInstrumentation SharedInvalCatalogMsg SharedInvalCatcacheMsg -SharedInvalRelSyncMsg +SharedInvalidationMessage SharedInvalRelcacheMsg SharedInvalRelmapMsg +SharedInvalRelSyncMsg SharedInvalSmgrMsg SharedInvalSnapshotMsg -SharedInvalidationMessage SharedJitInstrumentation SharedMemoizeInfo SharedRecordTableEntry SharedRecordTableKey SharedRecordTypmodRegistry SharedSeqScanInstrumentation +Sharedsort SharedSortInfo SharedTidRangeScanInstrumentation SharedTuplestore @@ -2968,10 +3543,20 @@ SharedTuplestoreAccessor SharedTuplestoreChunk SharedTuplestoreParticipant SharedTypmodTableEntry -Sharedsort +ShDependObjectInfo ShellTypeInfo ShippableCacheEntry ShippableCacheKey +shm_mq +shm_mq_handle +shm_mq_iovec +shm_mq_result +shm_toc +shm_toc_entry +shm_toc_estimator +shmem_hash_allocator +shmem_request_hook_type +shmem_startup_hook_type ShmemAllocatorData ShmemAttachCallback ShmemCallbacks @@ -2985,7 +3570,13 @@ ShmemStructOpts ShutdownForeignScan_function ShutdownInformation ShutdownMode +sig_atomic_t +siginfo_t +sigjmp_buf +signal_worker_item +signedbitmapword SignTSVector +sigset_t SimpleActionList SimpleActionListCell SimpleEcontextStackEntry @@ -2997,7 +3588,10 @@ SimpleStats SimpleStringList SimpleStringListCell SingleBoundSortItem +SinglePartitionSpec +SISeg Size +size_t SkipPages SkipSupport SkipSupportData @@ -3005,6 +3599,11 @@ SkipSupportIncDec SlabBlock SlabContext SlabSlot +slist_head +slist_iter +slist_mutable_iter +slist_node +slock_t SlotInvalidationCauseMap SlotNumber SlotSyncCtxStruct @@ -3021,6 +3620,10 @@ SlruShared SlruSharedData SlruWriteAll SlruWriteAllData +SMgrRelation +SMgrRelationData +SMgrSortArray +SN_local SnapBuild SnapBuildOnDisk SnapBuildState @@ -3028,12 +3631,15 @@ Snapshot SnapshotData SnapshotType SockAddr +socket_set +socklen_t Sort SortBy SortByDir SortByNulls SortCoordinate SortCoordinateData +SortedPoint SortGroupClause SortItem SortPath @@ -3043,17 +3649,26 @@ SortSupport SortSupportData SortTuple SortTupleComparator -SortedPoint +SpecialJoinInfo +SPELL +spgBulkDeleteState +spgChooseIn +spgChooseOut +spgChooseResultType +spgConfigIn +spgConfigOut +spgInnerConsistentIn +spgInnerConsistentOut SpGistBuildState SpGistCache SpGistDeadTuple SpGistDeadTupleData SpGistInnerTuple SpGistInnerTupleData -SpGistLUPCache SpGistLastUsedPage SpGistLeafTuple SpGistLeafTupleData +SpGistLUPCache SpGistMetaPageData SpGistNodeTuple SpGistNodeTupleData @@ -3065,15 +3680,45 @@ SpGistScanOpaqueData SpGistSearchItem SpGistState SpGistTypeDesc -SpecialJoinInfo +spgLeafConsistentIn +spgLeafConsistentOut +spgNodePtr +spgPickSplitIn +spgPickSplitOut +spgVacPendingItem +spgxlogAddLeaf +spgxlogAddNode +spgxlogMoveLeafs +spgxlogPickSplit +spgxlogSplitTuple +spgxlogState +spgxlogVacuumLeaf +spgxlogVacuumRedirect +spgxlogVacuumRoot +SPICallbackArg +SPIExecuteOptions SpinDelayStatus +SPIParseOpenOptions +SPIPlanPtr +SPIPrepareOptions +SPITupleTable +SPKFlushPosition +SPKRelMetaCacheEntry +split_pathtarget_context +split_pathtarget_item +SPLITCOST +SplitedPageLayout SplitInterval SplitLR SplitPageLayout SplitPoint SplitTextOutputData SplitVar -SplitedPageLayout +SPNode +SPNodeData +Spock_Stat_PendingSubEntry +Spock_Stat_StatSubEntry +Spock_Stat_Subscription SpockApplyChangeLogging SpockApplyProgress SpockApplyWorker @@ -3103,6 +3748,9 @@ SpockResolveOption SpockResourceDumpEvent SpockResourceDumpRec SpockSaveOriginConflictOption +spockStatsEntry +spockStatsKey +spockStatsType SpockSubscription SpockSyncEventMessage SpockSyncStatus @@ -3114,11 +3762,28 @@ SpockWalMessageSimple SpockWorker SpockWorkerStatus SpockWorkerType -Spock_Stat_PendingSubEntry -Spock_Stat_StatSubEntry -Spock_Stat_Subscription +SPPageDesc +sql_error_callback_arg +SQLDropObject +SQLFunctionCache +SQLFunctionCachePtr +SQLFunctionHashEntry +SQLFunctionParseInfo +SQLFunctionParseInfoPtr +sqlparseInfo +sqlparseState +SQLValueFunction +SQLValueFunctionOp +ss_lru_item_t +ss_scan_location_t +ss_scan_locations_t +ssize_t +SSL +SSL_CTX +SSLExtensionInfoContext StackElem StakindFlags +standard_qp_extra StartBufferIOResult StartDataPtrType StartLOPtrType @@ -3126,9 +3791,9 @@ StartLOsPtrType StartReplicationCmd StartupStatusEnum StatApproxReadStreamPrivate +StateFileChunk StatEntry StatExtEntry -StateFileChunk StatisticExtInfo StatsBuildData StatsData @@ -3137,42 +3802,54 @@ StatsExtInfo StdAnalyzeData StdRdOptIndexCleanup StdRdOptions -Step +stemmer_module +stmtCacheEntry StopList +storeInfo +storeRes_func StrategyNumber +stream_stop_callback StreamCtl StreamStopReason String StringInfo StringInfoData StripnullState +STRLEN SubLink SubLinkType SubOpts SubPlan SubPlanRTInfo SubPlanState -SubRelInfo -SubRemoveRels -SubTransactionId -SubXactCallback -SubXactCallbackItem -SubXactEvent -SubXactInfo SubqueryScan SubqueryScanPath SubqueryScanState SubqueryScanStatus +SubRelInfo +SubRemoveRels +SubscriberCreateContext SubscriptExecSetup SubscriptExecSteps -SubscriptRoutines -SubscriptTransform SubscriptingRef SubscriptingRefState Subscription SubscriptionInfo SubscriptionRelState SubscriptionTuple +SubscriptRoutines +SubscriptTransform +substitute_actual_parameters_context +substitute_actual_parameters_in_from_context +substitute_actual_srf_parameters_context +substitute_grouped_columns_context +substitute_phv_relids_context +SubTransactionId +SubXactCallback +SubXactCallbackItem +SubXactEvent +SubXactInfo +subxids_array_status SummarizerReadLocalXLogPrivate SupportRequestCost SupportRequestIndexCondition @@ -3184,83 +3861,25 @@ SupportRequestSelectivity SupportRequestSimplify SupportRequestSimplifyAggref SupportRequestWFuncMonotonic +SV +symbol Syn +SyncingRelationsState +SyncingTablesState SyncOps SyncRepConfigData SyncRepStandbyData SyncRequestHandler SyncRequestType SyncStandbySlotsConfigData -SyncingRelationsState -SyncingTablesState +SyscacheCallbackFunction SysCacheIdentifier SysFKRelationship SysScanDesc SysScanDescData -SyscacheCallbackFunction -SysloggerStartupData SystemRowsSamplerData SystemSamplerData SystemTimeSamplerData -TAPtype -TAR_MEMBER -TBMIterateResult -TBMIteratingState -TBMIterator -TBMPrivateIterator -TBMSharedIterator -TBMSharedIteratorState -TBMStatus -TBlockState -TCPattern -TIDBitmap -TM_FailureData -TM_IndexDelete -TM_IndexDeleteOp -TM_IndexStatus -TM_Result -TOKEN_DEFAULT_DACL -TOKEN_INFORMATION_CLASS -TOKEN_PRIVILEGES -TOKEN_USER -TParser -TParserCharTest -TParserPosition -TParserSpecial -TParserState -TParserStateAction -TParserStateActionItem -TQueueDestReceiver -TRGM -TSAnyCacheEntry -TSConfigCacheEntry -TSConfigInfo -TSDictInfo -TSDictionaryCacheEntry -TSExecuteCallback -TSLexeme -TSParserCacheEntry -TSParserInfo -TSQuery -TSQueryData -TSQueryParserState -TSQuerySign -TSReadPointer -TSTemplateInfo -TSTernaryValue -TSTokenTypeItem -TSTokenTypeStorage -TSVector -TSVectorBuildState -TSVectorData -TSVectorParseState -TSVectorStat -TState -TStatus -TStoreState -TU_UpdateIndexes -TXNEntryFile -TYPCATEGORY T_Action T_WorkerStatus TableAmRoutine @@ -3278,16 +3897,26 @@ TableScanDesc TableScanDescData TableScanInstrumentation TableSpaceCacheEntry -TableSpaceOpts -TableToProcess +tablespaceinfo TablespaceList TablespaceListCell +TableSpaceOpts +TableToProcess TapeBlockTrailer TapeShare +tar_file +TAR_MEMBER +TargetEntry TarMethodData TarMethodFile -TargetEntry -TclExceptionNameMap +TBlockState +TBMIterateResult +TBMIteratingState +TBMIterator +TBMPrivateIterator +TBMSharedIterator +TBMSharedIteratorState +TBMStatus Tcl_CmdInfo Tcl_DString Tcl_FileProc @@ -3298,19 +3927,29 @@ Tcl_NotifierProcs Tcl_Obj Tcl_Size Tcl_Time +TclExceptionNameMap +TCPattern +td_entry +temp_tablespaces_extra TempNamespaceStatus -TestDSMRegistryHashEntry -TestDSMRegistryStruct +teSection +test_re_flags +test_regex_ctx +test_shm_mq_header +test_spec TestDecodingData TestDecodingTxnData +TestDSMRegistryHashEntry +TestDSMRegistryStruct TestShmemData TestSlruSharedState -TestSpec TestValueType +text TextFreq TextPositionState TheLexeme TheSubstitute +TIDBitmap TidExpr TidExprType TidHashKey @@ -3325,41 +3964,66 @@ TidScanState TidStore TidStoreIter TidStoreIterResult +time_t TimeADT +timeKEY TimeLineHistoryCmd TimeLineHistoryEntry TimeLineID TimeOffset -TimeStamp -TimeTzADT -TimeZoneAbbrevTable +timeout_handler_proc +timeout_params TimeoutId TimeoutType Timestamp TimestampTz +TimeTzADT +TimeZoneAbbrevTable TimingClockSourceType +tlist_vinfo +TM_FailureData +TM_IndexDelete +TM_IndexDeleteOp +TM_IndexStatus +TM_Result TmFromChar TmToChar +toast_compress_header ToastAttrInfo ToastCompressionId -ToastTupleContext ToastedAttribute +ToastTupleContext TocEntry TokenAuxData +tokenize_error_callback_arg TokenizedAuthLine -TrackItem -TransApplyAction -TransInvalidationInfo -TransState -TransactionId -TransactionState -TransactionStateData +TParser +TParserCharTest +TParserPosition +TParserSpecial +TParserState +TParserStateAction +TParserStateActionItem +TQueueDestReceiver +TrackItem +TransactionId +TransactionState +TransactionStateData TransactionStmt TransactionStmtKind TransamVariablesData +TransApplyAction +transferMode TransformInfo TransformJsonStringValuesState +TransInvalidationInfo TransitionCaptureState +TransState +tree_mutator_callback +tree_walker_callback +trgm +TRGM +trgm_mb_char TrgmArc TrgmArcInfo TrgmBound @@ -3383,23 +4047,53 @@ TriggerFlags TriggerInfo TriggerInstrumentation TriggerTransition +trivalue TruncateStmt +ts_parserstate +ts_tokenizer +ts_tokentype +TSAnyCacheEntry TscClockSourceInfo +TSConfigCacheEntry +TSConfigInfo +TSDictInfo +TSDictionaryCacheEntry +tsearch_readline_state +TSExecuteCallback +tsKEY +TSLexeme TsmRoutine -TupOutputState -TupSortStatus -TupStoreStatus +TSParserCacheEntry +TSParserInfo +TSQuery +TSQueryData +TSQueryParserState +TSQuerySign +TSReadPointer +TState +TStatus +TSTemplateInfo +TSTernaryValue +TSTokenTypeItem +TSTokenTypeStorage +TStoreState +TSVector +TSVectorBuildState +TSVectorData +TSVectorParseState +TSVectorStat +TU_UpdateIndexes TupleConstr TupleConversionMap TupleDesc +tuplehash_hash +tuplehash_iterator TupleHashEntry TupleHashEntryData TupleHashIterator TupleHashTable TupleHashTableData TupleQueueReader -TupleTableSlot -TupleTableSlotOps TuplesortClusterArg TuplesortDatumArg TuplesortIndexArg @@ -3411,6 +4105,11 @@ TuplesortPublic TuplesortSpaceType Tuplesortstate Tuplestorestate +TupleTableSlot +TupleTableSlotOps +TupOutputState +TupSortStatus +TupStoreStatus TwoPhaseCallback TwoPhaseFileHeader TwoPhaseLockRecord @@ -3422,6 +4121,8 @@ TwoPhasePredicateXactRecord TwoPhaseRecordOnDisk TwoPhaseRmgrId TwoPhaseStateData +TXNEntryFile +TYPCATEGORY Type TypeCacheEntry TypeCacheEnumData @@ -3431,1199 +4132,167 @@ TypeFuncClass TypeInfo TypeName TzAbbrevCache +tzEntry +u_char +u_int U32 U8 +ua_page_items +ua_page_stats UCaseMap UChar UCharIterator -UColAttributeValue -UCollator -UConverter -UErrorCode -UINT -ULARGE_INTEGER -ULONG -ULONG_PTR -UV -UVersionInfo -UnicodeNormalizationForm -UnicodeNormalizationQC -Unique -UniquePath -UniquePathMethod -UniqueRelInfo -UniqueState -UnlistenStmt -UnresolvedTup -UnresolvedTupData -UpdateContext -UpdateStmt -UpgradeTask -UpgradeTaskProcessCB -UpgradeTaskReport -UpgradeTaskSlot -UpgradeTaskSlotState -UpgradeTaskStep -UploadManifestCmd -UpperRelationKind -UpperUniquePath -UserAuth -UserContext -UserMapping -UserOpts -VMCorruptionType -VacAttrStats -VacAttrStatsP -VacDeadItems -VacDeadItemsInfo -VacErrPhase -VacObjFilter -VacOptValue -VacuumCutoffs -VacuumParams -VacuumRelation -VacuumStmt -ValidIOData -ValidateIndexState -ValidatorModuleResult -ValidatorModuleState -ValidatorShutdownCB -ValidatorStartupCB -ValidatorValidateCB -ValuesScan -ValuesScanState -Var -VarBit -VarChar -VarParamState -VarReturningType -VarString -VarStringSortSupport -Variable -VariableAssignHook -VariableCache -VariableCacheData -VariableSetKind -VariableSetStmt -VariableShowStmt -VariableSpace -VariableStatData -VariableSubstituteHook -Variables -Vector32 -Vector8 -VersionedQuery -Vfd -ViewCheckOption -ViewOptCheckOption -ViewOptions -ViewStmt -VirtualTransactionId -VirtualTupleTableSlot -VolatileFunctionStatus -Vsrt -WAIT_ORDER -WALAvailability -WALInsertLock -WALInsertLockPadded -WALOpenSegment -WALReadError -WALSegmentCloseCB -WALSegmentContext -WALSegmentOpenCB -WCHAR -WCOKind -WFW_WaitOption -WIDGET -WORD -WORKSTATE -WSABUF -WSADATA -WSANETWORKEVENTS -WSAPROTOCOL_INFO -WaitEvent -WaitEventActivity -WaitEventBuffer -WaitEventBufferPin -WaitEventClient -WaitEventCustomCounterData -WaitEventCustomEntryByInfo -WaitEventCustomEntryByName -WaitEventIO -WaitEventIPC -WaitEventSet -WaitEventTimeout -WaitLSNProcInfo -WaitLSNResult -WaitLSNState -WaitLSNType -WaitPMResult -WaitStmt -WalCloseMethod -WalCompression -WalInsertClass -WalLevel -WalRcvData -WalRcvExecResult -WalRcvExecStatus -WalRcvState -WalRcvStreamOptions -WalRcvWakeupReason -WalReceiverConn -WalReceiverFunctionsType -WalSnd -WalSndCtlData -WalSndSendDataCallback -WalSndState -WalSummarizerData -WalSummaryFile -WalSummaryIO -WalTimeSample -WalUsage -WalWriteMethod -WalWriteMethodOps -Walfile -WindowAgg -WindowAggPath -WindowAggState -WindowAggStatus -WindowClause -WindowClauseSortData -WindowDef -WindowFunc -WindowFuncExprState -WindowFuncLists -WindowFuncRunCondition -WindowObject -WindowObjectData -WindowStatePerAgg -WindowStatePerAggData -WindowStatePerFunc -WindowStatePerFuncData -WithCheckOption -WithClause -WordBoundaryNext -WordEntry -WordEntryIN -WordEntryPos -WordEntryPosVector -WordEntryPosVector1 -WorkTableScan -WorkTableScanState -WorkerInfo -WorkerInfoData -WorkerInstrumentation -WorkerJobDumpPtrType -WorkerJobRestorePtrType -WorkerNodeInstrumentation -Working_State -WriteBufPtrType -WriteBytePtrType -WriteDataCallback -WriteDataPtrType -WriteExtraTocPtrType -WriteFunc -WriteManifestState -WriteTarState -WritebackContext -X509 -X509_EXTENSION -X509_NAME -X509_NAME_ENTRY -X509_STORE -X509_STORE_CTX -X86FeatureId -XLTW_Oper -XLogCtlData -XLogCtlInsert -XLogDumpConfig -XLogDumpPrivate -XLogLongPageHeader -XLogLongPageHeaderData -XLogPageHeader -XLogPageHeaderData -XLogPageReadCB -XLogPageReadPrivate -XLogPageReadResult -XLogPrefetchStats -XLogPrefetcher -XLogPrefetcherFilter -XLogReaderRoutine -XLogReaderState -XLogRecData -XLogRecPtr -XLogRecStats -XLogRecord -XLogRecordBlockCompressHeader -XLogRecordBlockHeader -XLogRecordBlockImageHeader -XLogRecordBuffer -XLogRecoveryCtlData -XLogRedoAction -XLogSegNo -XLogSource -XLogStats -XLogwrtResult -XLogwrtRqst -XPV -XPVIV -XPVMG -XactCallback -XactCallbackItem -XactEvent -XactLockTableWaitInfo -XidBoundsViolation -XidCacheStatus -XidCommitStatus -XidStatus -XmlExpr -XmlExprOp -XmlOptionType -XmlSerialize -XmlTableBuilderData -YYLTYPE -YYSTYPE -YY_BUFFER_STATE -ZSTD_CCtx -ZSTD_CStream -ZSTD_DCtx -ZSTD_DStream -ZSTD_cParameter -ZSTD_inBuffer -ZSTD_outBuffer -ZstdCompressorState -_SPI_connection -_SPI_plan -__m128i -__m512i -__mmask64 -__time64_t -_dev_t -_ino_t -_locale_t -_resultmap -_stringlist -access_vector_t -acquireLocksOnSubLinks_context -addFkConstraintSides -add_nulling_relids_context -adjust_appendrel_attrs_context -allocfunc -amadjustmembers_function -ambeginscan_function -ambuild_function -ambuildempty_function -ambuildphasename_function -ambulkdelete_function -amcanreturn_function -amcostestimate_function -amendscan_function -amestimateparallelscan_function -amgetbitmap_function -amgettreeheight_function -amgettuple_function -aminitparallelscan_function -aminsert_function -aminsertcleanup_function -ammarkpos_function -amoptions_function -amparallelrescan_function -amproperty_function -amrescan_function -amrestrpos_function -amtranslate_cmptype_function -amtranslate_strategy_function -amvacuumcleanup_function -amvalidate_function -array_iter -array_unnest_fctx -assign_collations_context -astreamer -astreamer_archive_context -astreamer_extractor -astreamer_gzip_decompressor -astreamer_gzip_writer -astreamer_lz4_frame -astreamer_member -astreamer_ops -astreamer_plain_writer -astreamer_recovery_injector -astreamer_tar_archiver -astreamer_tar_parser -astreamer_verify -astreamer_waldump -astreamer_zstd_frame -auth_password_hook_typ -auto_explain_extension_options -auto_explain_option -autovac_table -av_relation -avc_cache -avl_dbase -avl_node -avl_tree -avw_dbase -backslashResult -backup_file_entry -backup_file_hash -backup_manifest_info -backup_manifest_option -backup_wal_range -base_yy_extra_type -basebackup_options -bbsink -bbsink_copystream -bbsink_gzip -bbsink_lz4 -bbsink_ops -bbsink_server -bbsink_shell -bbsink_state -bbsink_throttle -bbsink_zstd -bbstreamer -bbstreamer_archive_context -bbstreamer_extractor -bbstreamer_gzip_decompressor -bbstreamer_gzip_writer -bbstreamer_lz4_frame -bbstreamer_member -bbstreamer_ops -bbstreamer_plain_writer -bbstreamer_recovery_injector -bbstreamer_tar_archiver -bbstreamer_tar_parser -bbstreamer_zstd_frame -bgworker_main_type -bh_node_type -binaryheap -binaryheap_comparator -bitmapword -bits16 -bits32 -bits8 -blockreftable_hash -blockreftable_iterator -bloom_filter -boolKEY -brin_column_state -brin_serialize_callback_type -btree_gin_convert_function -btree_gin_leftmost_function -build_simple_rel_hook_type -bytea -cached_re_str -canonicalize_state -cashKEY -catalogid_hash -cb_cleanup_dir -cb_options -cb_tablespace -cb_tablespace_mapping -char16_t -char32_t -check_agg_arguments_context -check_function_callback -check_network_data -check_object_relabel_type -check_password_hook_type -check_ungrouped_columns_context -child_process_kind -chr -cmpEntriesArg -codes_t -collation_cache_entry -collation_cache_hash -color -colormaprange -compare_context -config_handle -config_var_value -conn_errorMessage_func -conn_oauth_client_id_func -conn_oauth_client_secret_func -conn_oauth_discovery_uri_func -conn_oauth_issuer_id_func -conn_oauth_scope_func -conn_sasl_state_func -contain_aggs_of_level_context -contain_placeholder_references_context -convert_testexpr_context -copy_data_dest_cb -copy_data_source_cb -core_YYSTYPE -core_yy_extra_type -core_yyscan_t -corrupt_items -cost_qual_eval_context -count_param_references_context -cp_hash_func -create_upper_paths_hook_type -createdb_failure_params -crosstab_HashEnt -crosstab_cat_desc -curl_infotype -curl_socket_t -curl_version_info_data -datapagemap_iterator_t -datapagemap_t -dateKEY -datetkn -dce_uuid_t -dclist_head -decimal -deparse_columns -deparse_context -deparse_expr_cxt -deparse_namespace -derives_hash -destructor -dev_t -digit -disassembledLeaf -dlist_head -dlist_iter -dlist_mutable_iter -dlist_node -dm_code -dm_codes -dm_letter -dm_node -ds_state -dsa_area -dsa_area_control -dsa_area_pool -dsa_area_span -dsa_handle -dsa_pointer -dsa_pointer_atomic -dsa_segment_header -dsa_segment_index -dsa_segment_map -dshash_compare_function -dshash_copy_function -dshash_hash -dshash_hash_function -dshash_parameters -dshash_partition -dshash_seq_status -dshash_table -dshash_table_control -dshash_table_handle -dshash_table_item -dsm_control_header -dsm_control_item -dsm_handle -dsm_op -dsm_segment -dsm_segment_detach_callback -duration -eLogType -ean13 -eary -ec_matches_callback_type -ec_member_foreign_arg -ec_member_matches_arg -element_type -emit_log_hook_type -eval_const_expressions_context -exec_thread_arg -execution_state -exit_function -explain_get_index_name_hook_type -explain_per_node_hook_type -explain_per_plan_hook_type -explain_validate_options_hook_type -f_smgr -fasthash_state -fd_set -fe_oauth_state -fe_scram_state -fe_scram_state_enum -fetch_range_request -file_action_t -file_content_type_t -file_entry_t -file_type_t -filehash_hash -filehash_iterator -filemap_t -fill_string_relopt -finalize_primnode_context -find_dependent_phvs_context -find_expr_references_context -fireRIRonSubLink_context -fix_join_expr_context -fix_scan_expr_context -fix_upper_expr_context -fix_windowagg_cond_context -flatten_join_alias_vars_context -flatten_rtes_walker_context -float4 -float4KEY -float8 -float8KEY -floating_decimal_32 -floating_decimal_64 -fmAggrefPtr -fmExprContextCallbackFunction -fmNodePtr -fmStringInfo -fmgr_hook_type -foreign_glob_cxt -foreign_loc_cxt -freefunc -fsec_t -gbt_vsrt_arg -gbtree_ninfo -gbtree_vinfo -generate_series_fctx -generate_series_numeric_fctx -generate_series_timestamp_fctx -generate_series_timestamptz_fctx -generate_subscripts_fctx -get_attavgwidth_hook_type -get_index_stats_hook_type -get_relation_info_hook_type -get_relation_stats_hook_type -gid_t -gin_leafpage_items_state -ginxlogCreatePostingTree -ginxlogDeleteListPages -ginxlogDeletePage -ginxlogInsert -ginxlogInsertDataInternal -ginxlogInsertEntry -ginxlogInsertListPage -ginxlogRecompressDataLeaf -ginxlogSplit -ginxlogUpdateMeta -ginxlogVacuumDataLeafPage -gistxlogDelete -gistxlogPage -gistxlogPageDelete -gistxlogPageReuse -gistxlogPageSplit -gistxlogPageUpdate -grouping_eqop_callback -grouping_sets_data -grouping_walker_ctx -growable_trgm_array -gseg_picksplit_item -gss_OID_set -gss_buffer_desc -gss_cred_id_t -gss_cred_usage_t -gss_ctx_id_t -gss_key_value_element_desc -gss_key_value_set_desc -gss_name_t -gtrgm_consistent_cache -gzFile -hashfunc -having_grouping_ctx -hbaPort -heap_page_items_state -help_handler -hlCheck -host_cache_hash -hstoreCheckKeyLen_t -hstoreCheckValLen_t -hstorePairs_t -hstoreUniquePairs_t -hstoreUpgrade_t -hyperLogLogState -ifState -import_error_callback_arg -indexed_tlist -inet -inetKEY -inet_struct -initRowMethod -init_function -inline_cte_walker_context -inline_error_callback_arg -ino_t -inquiry -instr_time -int128 -int16 -int16KEY -int16_t -int2vector -int32 -int32KEY -int32_t -int64 -int64KEY -int64_t -int8 -int8_t -int8x16_t -int_fast32_t -int_fast64_t -internalPQconninfoOption -intmax_t -intptr_t -intset_internal_node -intset_leaf_node -intset_node -intvKEY -io_callback_fn -io_stat_col -itemIdCompact -itemIdCompactData -iterator -jmp_buf -join_path_setup_hook_type -join_search_hook_type -joinrel_setup_hook_type -json_aelem_action -json_manifest_error_callback -json_manifest_per_file_callback -json_manifest_per_wal_range_callback -json_manifest_perfile_callback -json_manifest_perwalrange_callback -json_manifest_system_identifier_callback -json_manifest_version_callback -json_ofield_action -json_scalar_action -json_struct_action -keepwal_entry -keepwal_hash -keyEntryData -key_t -lclContext -lclTocEntry -leafSegmentInfo -leaf_item -libpq_gettext_func -libpq_source -libpqsrv_PGresult -line_t -lineno_t -list_sort_comparator -loc_chunk -local_relopt -local_relopts -local_source -local_ts_iter -local_ts_radix_tree -locale_t -locate_agg_of_level_context -locate_var_of_level_context -locate_windowfunc_context -logstreamer_param -lquery -lquery_level -lquery_variant -ltree -ltree_gist -ltree_level -ltxtquery -mXactCacheEnt -mac8KEY -macKEY -macaddr -macaddr8 -macaddr_sortsupport_state -manifest_data -manifest_file -manifest_files_hash -manifest_files_iterator -manifest_wal_range -manifest_writer -map_variable_attnos_context -max_parallel_hazard_context -mb2wchar_with_len_converter -mbchar_verifier -mbcharacter_incrementer -mbdisplaylen_converter -mblen_converter -mbstr_verifier -memoize_hash -memoize_iterator -metastring -missing_cache_key -mix_data_t -mixedStruct -mode_t -movedb_failure_params -multirange_bsearch_comparison -multirange_unnest_fctx -mxact -mxtruncinfo -needs_fmgr_hook_type -network_sortsupport_state -nl_item -nodeitem -normal_rand_fctx -nsphash_hash -ntile_context -nullingrel_info -numeric -object_access_hook_type -object_access_hook_type_str -off_t -oidKEY -oidvector -on_dsm_detach_callback -on_exit_nicely_callback -openssl_tls_init_hook_typ -ossl_EVP_cipher_func -other -output_type -overexplain_options -pagetable_hash -pagetable_iterator -pairingheap -pairingheap_comparator -pairingheap_node -pam_handle_t -parallel_worker_main_type -parse_error_callback_arg -parser_context -partition_method_t -pe_test_config -pe_test_escape_func -pe_test_vector -pendingPosition -pending_label -pgParameterStatus -pg_atomic_flag -pg_atomic_uint32 -pg_atomic_uint64 -pg_be_sasl_mech -pg_case_map -pg_category_range -pg_checksum_context -pg_checksum_raw_context -pg_checksum_type -pg_compress_algorithm -pg_compress_specification -pg_conn_host -pg_conn_host_type -pg_conv_map -pg_crc32 -pg_crc32c -pg_cryptohash_ctx -pg_cryptohash_errno -pg_cryptohash_type -pg_ctype_cache -pg_enc -pg_enc2gettext -pg_enc2name -pg_encname -pg_fe_sasl_mech -pg_funcptr_t -pg_getopt_ctx -pg_gssinfo -pg_hmac_ctx -pg_hmac_errno -pg_int64 -pg_local_to_utf_combined -pg_locale_t -pg_mb_radix_tree -pg_md5_ctx -pg_on_exit_callback -pg_plan_advice_advisor_hook -pg_prng_state -pg_re_flags -pg_regex_t -pg_regmatch_t -pg_regoff_t -pg_saslprep_rc -pg_saslprep_test_context -pg_sha1_ctx -pg_sha224_ctx -pg_sha256_ctx -pg_sha384_ctx -pg_sha512_ctx -pg_signal_info -pg_snapshot -pg_special_case -pg_stack_base_t -pg_ternary -pg_time_t -pg_time_usec_t -pg_tz -pg_tz_cache -pg_tzenum -pg_unicode_category -pg_unicode_decompinfo -pg_unicode_decomposition -pg_unicode_norminfo -pg_unicode_normprops -pg_unicode_properties -pg_unicode_range -pg_unicode_recompinfo -pg_usec_time_t -pg_utf8_codepoint_range -pg_utf_to_local_combined -pg_uuid_t -pg_wc_probefunc -pg_wchar -pg_wchar_tbl -pgoff_t -pgp_armor_headers_state -pgpa_advice_item -pgpa_advice_tag_type -pgpa_advice_target -pgpa_identifier -pgpa_index_target -pgpa_itm_type -pgpa_jo_outcome -pgpa_join_member -pgpa_join_state -pgpa_join_strategy -pgpa_join_unroller -pgpa_output_context -pgpa_plan_walker_context -pgpa_planner_info -pgpa_planner_state -pgpa_qf_type -pgpa_query_feature -pgpa_scan -pgpa_scan_strategy -pgpa_target_type -pgpa_trove -pgpa_trove_entry -pgpa_trove_entry_element -pgpa_trove_entry_hash -pgpa_trove_entry_key -pgpa_trove_lookup_type -pgpa_trove_result -pgpa_trove_slice -pgpa_unrolled_join -pgpa_yy_extra_type -pgsa_entry -pgsa_entry_key -pgsa_saved_entry -pgsa_saved_stash -pgsa_saved_stash_table_hash -pgsa_saved_stash_table_iterator -pgsa_shared_state -pgsa_stash -pgsa_stash_count -pgsa_stash_count_table_hash -pgsa_stash_name -pgsa_stash_name_table_hash -pgsa_writer_context -pgsocket -pgsql_thing_t -pgssEntry -pgssGlobalStats -pgssHashKey -pgssSharedState -pgssStoreKind -pgssVersion -pgstat_entry_ref_hash_hash -pgstat_entry_ref_hash_iterator -pgstat_page -pgstat_snapshot_hash -pgstattuple_type -pgthreadlock_t -pid_t -pivot_field -planner_hook_type -planner_setup_hook_type -planner_shutdown_hook_type -planstate_tree_walker_callback -plperl_array_info -plperl_call_data -plperl_interp_desc -plperl_proc_desc -plperl_proc_key -plperl_proc_ptr -plperl_query_desc -plperl_query_entry -plpgsql_CastExprHashEntry -plpgsql_CastHashEntry -plpgsql_CastHashKey -plpgsql_HashEnt -plpgsql_expr_walker_callback -plpgsql_stmt_walker_callback -pltcl_call_state -pltcl_interp_desc -pltcl_proc_desc -pltcl_proc_key -pltcl_proc_ptr -pltcl_query_desc -pointer -polymorphic_actuals -pos_trgm -post_parse_analyze_hook_type -postprocess_result_function -pqbool -pqsigfunc -pqsigfunc_legacy -printQueryOpt -printTableContent -printTableFooter -printTableOpt -printTextFormat -printTextLineFormat -printTextLineWrap -printTextRule -printXheaderWidthType -printfunc -priv_map -process_file_callback_t -process_sublinks_context -proclist_head -proclist_mutable_iter -proclist_node -promptStatus_t -pthread_barrier_t -pthread_cond_t -pthread_key_t -pthread_mutex_t -pthread_once_t -pthread_t -ptrdiff_t -publication_tables_state -published_rel -pull_var_clause_context -pull_varattnos_context -pull_varnos_context -pull_vars_context -pullup_replace_vars_context -pushdown_safe_type -pushdown_safety_info -qc_hash_func -qsort_arg_comparator -qsort_comparator -query_pathkeys_callback -radius_attribute -radius_packet -rangeTableEntry_used_context -rank_context -rbt_allocfunc -rbt_combiner -rbt_comparator -rbt_freefunc -reduce_outer_joins_partial_state -reduce_outer_joins_pass1_state -reduce_outer_joins_pass2_state -refcount_hash -refcount_iterator -reference -regc_wc_probefunc -regex_arc_t -regex_t -regexp -regexp_matches_ctx -registered_buffer -regmatch_t -regoff_t -regproc -relopt_bool -relopt_enum -relopt_enum_elt_def -relopt_gen -relopt_int -relopt_kind -relopt_parse_elt -relopt_real -relopt_string -relopt_ternary -relopt_type -relopt_value -relopts_validator -remoteConn -remoteConnHashEnt -remoteDep -remove_nulling_relids_context -rendezvousHashEntry -rep -replace_rte_variables_callback -replace_rte_variables_context -report_error_fn -ret_type -rewind_source -rewrite_event -rf_context -rfile -rm_detail_t -role_auth_extra -rolename_hash -row_security_policy_hook_type -rsv_callback -rt_iter -rt_node_class_test_elem -rt_radix_tree -saophash_hash -save_buffer -save_locale_t -scram_state -scram_state_enum -script_error_callback_arg -security_class_t -sem_t -sepgsql_context_info_t -sequence_magic -set_conn_altsock_func -set_conn_oauth_token_func -set_join_pathlist_hook_type -set_rel_pathlist_hook_type -shared_ts_iter -shared_ts_radix_tree -shm_mq -shm_mq_handle -shm_mq_iovec -shm_mq_result -shm_toc -shm_toc_entry -shm_toc_estimator -shmem_hash_allocator -shmem_request_hook_type -shmem_startup_hook_type -sig_atomic_t -siginfo_t -sigjmp_buf -signal_worker_item -signedbitmapword -sigset_t -size_t -slist_head -slist_iter -slist_mutable_iter -slist_node -slock_t -socket_set -socklen_t -spgBulkDeleteState -spgChooseIn -spgChooseOut -spgChooseResultType -spgConfigIn -spgConfigOut -spgInnerConsistentIn -spgInnerConsistentOut -spgLeafConsistentIn -spgLeafConsistentOut -spgNodePtr -spgPickSplitIn -spgPickSplitOut -spgVacPendingItem -spgxlogAddLeaf -spgxlogAddNode -spgxlogMoveLeafs -spgxlogPickSplit -spgxlogSplitTuple -spgxlogState -spgxlogVacuumLeaf -spgxlogVacuumRedirect -spgxlogVacuumRoot -split_pathtarget_context -split_pathtarget_item -spockStatsEntry -spockStatsKey -spockStatsType -sql_error_callback_arg -sqlparseInfo -sqlparseState -ss_lru_item_t -ss_scan_location_t -ss_scan_locations_t -ssize_t -standard_qp_extra -stemmer_module -stmtCacheEntry -storeInfo -storeRes_func -stream_stop_callback -string -substitute_actual_parameters_context -substitute_actual_parameters_in_from_context -substitute_actual_srf_parameters_context -substitute_grouped_columns_context -substitute_phv_relids_context -subxids_array_status -symbol -tablespaceinfo -tar_file -td_entry -teSection -temp_tablespaces_extra -test128 -test_re_flags -test_regex_ctx -test_shm_mq_header -test_spec -test_start_function -text -timeKEY -time_t -timeout_handler_proc -timeout_params -timerCA -tlist_vinfo -toast_compress_header -tokenize_error_callback_arg -transferMode -transfer_thread_arg -tree_mutator_callback -tree_walker_callback -trgm -trgm_mb_char -trivalue -tsKEY -ts_parserstate -ts_tokenizer -ts_tokentype -tsearch_readline_state -tuplehash_hash -tuplehash_iterator -type -tzEntry -u_char -u_int -ua_page_items -ua_page_stats uchr +UColAttributeValue +UCollator +UConverter +UErrorCode uid_t +uint_fast64_t uint128 uint16 uint16_t -uint16x8_t uint32 uint32_t -uint32x4_t uint64 uint64_t -uint64x2_t uint8 uint8_t -uint8x16_t -uint_fast64_t uintmax_t uintptr_t +unicode_linestyle +UnicodeNormalizationForm +UnicodeNormalizationQC unicodeStyleBorderFormat unicodeStyleColumnFormat unicodeStyleFormat unicodeStyleRowFormat -unicode_linestyle +Unique +UniquePath +UniquePathMethod +UniqueRelInfo +UniqueState unit_conversion +UnlistenStmt unlogged_relation_entry +UnresolvedTup +UnresolvedTupData +UpdateContext +UpdateStmt +UpgradeTask +UpgradeTaskProcessCB +UpgradeTaskReport +UpgradeTaskSlot +UpgradeTaskSlotState +UpgradeTaskStep +UploadManifestCmd +UpperRelationKind +UpperUniquePath +UserAuth +UserContext +UserMapping +UserOpts utf_local_conversion_func -uuidKEY uuid_rc_t uuid_sortsupport_state uuid_t +uuidKEY +UV +UVersionInfo va_list +VacAttrStats +VacAttrStatsP +VacDeadItems +VacDeadItemsInfo +VacErrPhase +VacObjFilter +VacOptValue +VacuumCutoffs vacuumingOptions +VacuumParams +VacuumRelation +VacuumStmt validate_string_relopt +ValidateIndexState +ValidatorModuleResult +ValidatorModuleState +ValidatorShutdownCB +ValidatorStartupCB +ValidatorValidateCB +ValidIOData +ValuesScan +ValuesScanState +Var varatt_expanded varatt_external varatt_indirect varattrib_1b varattrib_1b_e varattrib_4b +VarBit +VarChar +Variable +VariableAssignHook +VariableCache +VariableCacheData +Variables +VariableSetKind +VariableSetStmt +VariableShowStmt +VariableSpace +VariableStatData +VariableSubstituteHook varlena +VarParamState +VarReturningType +VarString +VarStringSortSupport vartag_external vbits +Vector32 +Vector8 +VerbosityLevelEnum verifier_context +VersionedQuery +Vfd +ViewCheckOption +ViewOptCheckOption +ViewOptions +ViewStmt +VirtualTransactionId +VirtualTupleTableSlot +VMCorruptionType +VolatileFunctionStatus +Vsrt +WAIT_ORDER +WaitEvent +WaitEventActivity +WaitEventBuffer +WaitEventBufferPin +WaitEventClient +WaitEventCustomCounterData +WaitEventCustomEntryByInfo +WaitEventCustomEntryByName +WaitEventIO +WaitEventIPC +WaitEventSet +WaitEventTimeout +WaitLSNProcInfo +WaitLSNResult +WaitLSNState +WaitLSNType +WaitPMResult +WaitStmt +WaitTracker +WALAvailability +WalCloseMethod +WalCompression +Walfile +WalInsertClass +WALInsertLock +WALInsertLockPadded +WalLevel +WALOpenSegment walrcv_alter_slot_fn walrcv_check_conninfo_fn walrcv_connect_fn @@ -4641,15 +4310,98 @@ walrcv_receive_fn walrcv_send_fn walrcv_server_version_fn walrcv_startstreaming_fn -wchar2mb_with_len_converter +WalRcvData +WalRcvExecResult +WalRcvExecStatus +WalRcvState +WalRcvStreamOptions +WalRcvWakeupReason +WALReadError +WalReceiverConn +WalReceiverFunctionsType +WALSegmentCloseCB +WALSegmentContext +WALSegmentOpenCB +WalSnd +WalSndCtlData +WalSndSendDataCallback +WalSndState +WalSummarizerData +WalSummaryFile +WalSummaryIO +WalTimeSample +WalUsage +WalWriteMethod +WalWriteMethodOps wchar_t -win32_deadchild_waitinfo +wchar2mb_with_len_converter +WCOKind +WFW_WaitOption +WIDGET +WindowAgg +WindowAggPath +WindowAggState +WindowAggStatus +WindowClause +WindowClauseSortData +WindowDef +WindowFunc +WindowFuncExprState +WindowFuncLists +WindowFuncRunCondition +WindowObject +WindowObjectData +WindowStatePerAgg +WindowStatePerAggData +WindowStatePerFunc +WindowStatePerFuncData wint_t +WithCheckOption +WithClause +WordBoundaryNext +WordEntry +WordEntryIN +WordEntryPos +WordEntryPosVector +WordEntryPosVector1 worker_state +WorkerInfo +WorkerInfoData +WorkerInstrumentation +WorkerJobDumpPtrType +WorkerJobRestorePtrType +WorkerNodeInstrumentation +Working_State +WORKSTATE worktable -wrap +WorkTableScan +WorkTableScanState +WritebackContext +WriteBufPtrType +WriteBytePtrType +WriteDataCallback +WriteDataPtrType +WriteExtraTocPtrType +WriteFunc +WriteManifestState +WriteTarState ws_file_info ws_options +X509 +X509_EXTENSION +X509_NAME +X509_NAME_ENTRY +X509_STORE +X509_STORE_CTX +X86FeatureId +XactCallback +XactCallbackItem +XactEvent +XactLockTableWaitInfo +XidBoundsViolation +XidCacheStatus +XidCommitStatus +XidStatus xl_brin_createidx xl_brin_desummarize xl_brin_insert @@ -4744,35 +4496,83 @@ xl_xact_xinfo xlhp_freeze_plan xlhp_freeze_plans xlhp_prune_items +XLogCtlData +XLogCtlInsert +XLogDumpConfig +XLogDumpPrivate +XLogLongPageHeader +XLogLongPageHeaderData +XLogPageHeader +XLogPageHeaderData +XLogPageReadCB +XLogPageReadPrivate +XLogPageReadResult +XLogPrefetcher +XLogPrefetcherFilter +XLogPrefetchStats +XLogReaderRoutine +XLogReaderState +XLogRecData +XLogRecord +XLogRecordBlockCompressHeader +XLogRecordBlockHeader +XLogRecordBlockImageHeader +XLogRecordBuffer +XLogRecoveryCtlData +XLogRecPtr +XLogRecStats +XLogRedoAction +XLogSegNo +XLogSource +XLogStats +XLogwrtResult +XLogwrtRqst +XLTW_Oper xmlBuffer xmlBufferPtr xmlChar xmlDocPtr xmlError -xmlErrorPtr +XmlExpr +XmlExprOp xmlExternalEntityLoader xmlGenericErrorFunc xmlNodePtr xmlNodeSetPtr +XmlOptionType xmlParserCtxtPtr -xmlParserErrors xmlParserInputPtr xmlSaveCtxt xmlSaveCtxtPtr +XmlSerialize xmlStructuredErrorFunc +XmlTableBuilderData xmlTextWriter xmlTextWriterPtr +xmltype xmlXPathCompExprPtr xmlXPathContextPtr xmlXPathObjectPtr -xmltype xpath_workspace +XPV +XPVIV +XPVMG xsltSecurityPrefsPtr xsltStylesheetPtr xsltTransformContextPtr +YY_BUFFER_STATE yy_parser yy_size_t +YYLTYPE yyscan_t +YYSTYPE z_stream z_streamp -zic_t +ZSTD_CCtx +ZSTD_cParameter +ZSTD_CStream +ZSTD_DCtx +ZSTD_DStream +ZSTD_inBuffer +ZSTD_outBuffer +ZstdCompressorState diff --git a/utils/spock_create_subscriber/Makefile b/utils/spock_create_subscriber/Makefile new file mode 100644 index 00000000..54e39446 --- /dev/null +++ b/utils/spock_create_subscriber/Makefile @@ -0,0 +1,15 @@ +# Makefile for spock_create_subscriber utility +PG_CONFIG ?= pg_config +PROGRAM = spock_create_subscriber + +PG_CPPFLAGS = -I../../include -I$(shell $(PG_CONFIG) --includedir) +PG_LDFLAGS = -lpq -L$(shell $(PG_CONFIG) --libdir) + +# create symlink to spock_fe.c here +spock_fe.c: ../../src/spock_fe.c + ln -sf $< $@ +OBJS = spock_create_subscriber.o spock_fe.o + +# PGXS +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c new file mode 100644 index 00000000..f2c9fc6c --- /dev/null +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -0,0 +1,6845 @@ +/* ------------------------------------------------------------------------- + * + * spock_create_subscriber.c + * Initialize a new spock subscriber from a physical base backup + * + * Copyright (c) 2022-2024, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + * ------------------------------------------------------------------------- + */ + +/* dirent.h on port/win32_msvc expects MAX_PATH to be defined */ +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Note the order is important for debian here. */ +#if !defined(pg_attribute_printf) + +/* GCC and XLC support format attributes */ +#if defined(__GNUC__) || defined(__IBMC__) +#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) +#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) +#else +#define pg_attribute_format_arg(a) +#define pg_attribute_printf(f,a) +#endif + +#endif + +#include "libpq-fe.h" +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +#include "getopt_long.h" + +#include "miscadmin.h" + +#include "access/timeline.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "common/controldata_utils.h" +#include "common/file_utils.h" +#include "common/jsonapi.h" +#include "common/logging.h" +#include "mb/pg_wchar.h" +#include "port.h" + +#include "spock_fe.h" + +#define MAX_APPLY_DELAY 86400 + +typedef struct RemoteInfo +{ + Oid nodeid; + char *node_name; + char *sysid; + char *dbname; + char *replication_sets; + TimeLineID timeline_id; /* current TLI, for detecting a data_dir left + * over from an already-promoted earlier + * attempt (see check_data_dir()) */ +} RemoteInfo; + +typedef struct PeerNodeInfo +{ + char *node_name; + char *dsn; + char *slot_name; /* from spock.spock_gen_slot_name() */ + char *sub_name; /* "sub__" */ + bool reverse_sub_created; + + /* + * slot_creation_lsn doubles as the "peer slot created" flag: a non-empty + * value means the slot exists, so no separate boolean is kept. It is + * persisted to the manifest for --cleanup; peer_marker_lsn below it is + * in-memory only -- it doesn't gate a --cleanup decision, and cutover has + * no cross-invocation resume in v1 (see check_reused_data_dir_is_safe()). + */ + char *slot_creation_lsn; /* peer's replication slot creation LSN */ + char *peer_marker_lsn; /* this peer's marker once seen on the + * source */ +} PeerNodeInfo; + +typedef struct BidirectionalState +{ + bool enabled; + int num_peers; + PeerNodeInfo *peers; + int stall_timeout; /* default 600s */ + int max_wait; /* default 0 = unbounded */ + char *source_slot_name; + char *source_restore_lsn; /* recovery target LSN; consumed by the + * disabled-first catchup sub_create. + * Written to the manifest as a diagnostic + * record of what was used, but never read + * back on a --cleanup reload -- nothing + * needs it restored into memory there */ + char *node_dsn; /* DSN registered via spock.node_create(); the + * address peers use to connect back to this + * node. Derived from --subscriber-dsn. */ + char *node_sysid; /* n3's system_identifier; lets --cleanup + * confirm node_dsn still reaches this node + * before dropping subscriptions there. */ + bool source_reverse_sub_created; /* sub__ created on + * the source -- must be + * persisted, unlike the peer + * progress bits, since it is the + * only way --cleanup knows to + * drop this specific subscription + * on a node that also has other, + * unrelated, legitimate + * subscriptions */ + bool cleanup_mode; + bool force_cleanup; /* --force: also remove the data directory on + * --cleanup, not just remote state */ + char *manifest_path; +} BidirectionalState; + +/* + * Replication-set / table-membership / sequence state captured from the + * source's catalog before DROP EXTENSION spock removes it. Utility-side + * memory only; never written to the manifest. + */ +typedef struct RepsetCapture +{ + char *set_name; + bool replicate_insert; + bool replicate_update; + bool replicate_delete; + bool replicate_truncate; +} RepsetCapture; + +typedef struct RepsetTableCapture +{ + char *set_name; + char *qualified_table; /* rts.set_reloid::regclass */ + char *columns; /* rts.set_att_list, NULL if all columns */ + char *row_filter; /* pg_get_expr(...), NULL if none */ +} RepsetTableCapture; + +typedef struct SequenceCapture +{ + char *set_name; + char *qualified_seq; + int64 last_value; + bool is_called; +} SequenceCapture; + +typedef struct CatalogCapture +{ + RepsetCapture *repsets; + int num_repsets; + RepsetTableCapture *tables; + int num_tables; + SequenceCapture *sequences; + int num_sequences; +} CatalogCapture; + +/* + * main()'s working state, threaded by pointer through each lifecycle + * phase function below (parse_options() through + * restart_with_spock_and_activate()) in the same order those phases run. + * Holds every option and every piece of state one phase sets that a + * later phase reads; pure loop/scratch locals (loop counters, one-shot + * PQExpBuffers, pg_ctl_ret) stay local to whichever phase uses them. + */ +typedef struct SubscriberCreateContext +{ + /* Parsed options */ + char *subscriber_name; + char *base_sub_connstr; + char *base_prov_connstr; + char *replication_sets; + char *databases; + char *postgresql_conf; + char *postgresql_auto_conf; + char *pg_hba_conf; + char *recovery_conf; + int apply_delay; + bool force_text_transfer; + bool stop; + bool drop_slot_if_exists; + char *extra_basebackup_args; + BidirectionalState bidir; + char bidir_manifest_path[MAXPGPATH]; + char bidir_pending_path[MAXPGPATH]; + + /* Resolved database targets */ + char **database_list; + int n_databases; + + /* State set by one phase and consumed by a later one */ + char **slot_names; + char *sub_connstr; + char *prov_connstr; + bool use_existing_data_dir; + RemoteInfo *remote_info; + char *remote_lsn; + CatalogCapture capture; +} SubscriberCreateContext; + +typedef enum +{ + VERBOSITY_NORMAL, + VERBOSITY_VERBOSE, + VERBOSITY_DEBUG +} VerbosityLevelEnum; + +static char *argv0 = NULL; +static const char *progname; +static char *data_dir = NULL; +static char pid_file[MAXPGPATH]; +static time_t start_time; +static VerbosityLevelEnum verbosity = VERBOSITY_NORMAL; + +/* defined as static so that die() can close them */ +static PGconn *subscriber_conn = NULL; +static PGconn *provider_conn = NULL; + +static void signal_handler(int sig); +static void usage(void); +static void die(const char *fmt,...) + pg_attribute_printf(1, 2); +static void print_msg(VerbosityLevelEnum level, const char *fmt,...) + pg_attribute_printf(2, 3); +static PGresult *debug_exec(PGconn *conn, const char *query); + +static int run_pg_ctl(const char *arg); +static void validate_extra_basebackup_args(const char *args); +static int parse_checked_int(const char *arg, const char *opt_name); +static char *validated_existing_path(const char *arg, const char *file_label); +static void run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args); +static char *reset_subscriber_sysid(const char *data_dir); +static void run_pg_resetwal(const char *data_dir); +static void wait_postmaster_connection(const char *connstr); +static void wait_primary_connection(const char *connstr, int stall_timeout, int max_wait); +static void wait_postmaster_shutdown(void); + +static char *validate_replication_set_input(char *replication_sets); + +static void remove_unwanted_data(PGconn *conn); +static void initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn); +static char *create_restore_point(PGconn *conn, char *restore_point_name); +static char *create_logical_slot(PGconn *conn, const char *slot_name, + bool drop_if_exists, const char *already_exists_hint); +static char *initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists); +static char *create_peer_slot(PGconn *peer_conn, const char *peer_slot_name); +static char *sub_name_for(const char *local_node_name, const char *provider_node_name); +static char *get_origin_name_for_node(PGconn *conn, const char *upstream_node_name, + const char *conn_label, char **sub_name_out); +static void spock_subscribe(PGconn *conn, char *subscriber_name, + char *subscriber_dsn, + char *provider_connstr, + char *replication_sets, + int apply_delay, + bool force_text_transfer); + +static RemoteInfo *get_remote_info(PGconn *conn); + +static bool extension_exists(PGconn *conn, const char *extname); +static void install_extension(PGconn *conn, const char *extname); + +static void ensure_trailing_newline(const char *path); +static void initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args); +static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); +static void check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo); + +static char *read_sysid(const char *data_dir); + +static void WriteRecoveryConf(PQExpBuffer contents); +static void CopyConfFile(char *fromfile, char *tofile, bool append); + +static char *get_connstr_dbname(char *connstr); +static char *get_connstr(char *connstr, char *dbname); +static char *PQconninfoParamsToConnstr(const char *const *keywords, const char *const *values); +static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); + +static bool file_exists(const char *path); +static char *expand_tilde(char *path); +static bool is_pg_dir(const char *path); +static void copy_file(char *fromfile, char *tofile, bool append); +static char *find_other_exec_or_die(const char *argv0, const char *target); +static bool postmaster_is_alive(pid_t pid); +static long get_pgpid(void); +static char **get_database_list(char *databases, int *n_databases); +static char *generate_restore_point_name(void); + +static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out); +static void check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers); +static void check_spock_version_at_least_6(PGconn *conn, const char *node_label); +static void check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes); +static void check_peer_identity(PGconn *peer_conn, const char *expected_name); +static void check_replication_set_equivalence(PGconn *source_conn, + PeerNodeInfo *peers, int num_peers); +static void write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn); +static bool read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out); +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir); +static void stop_postgres_in_data_dir(void); +static bool remove_data_dir_if_forced(bool force); +static bool check_sysid_matches(PGconn *conn, const char *expected_sysid); +static void append_json_string(PQExpBuffer buf, const char *str); + +static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname); +static void check_no_native_subscriptions(PGconn *conn); +static void capture_catalog_state(PGconn *conn, Oid source_nodeid, + CatalogCapture *capture); +static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); +static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); +static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn); +static void create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets); +static char *get_sync_event_lsn(PGconn *conn, const char *node_label); +static char *get_catchup_target_lsn(const char *source_dsn); +static bool apply_worker_is_busy(PGconn *conn, const char *sub_name); +static void wait_for_origin_progress(PGconn *conn, const char *origin_name, + const char *target_lsn, const char *watch_sub_name, + const char *context_label, int stall_timeout, int max_wait); +static void wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait); +static void establish_peer_coverage_barrier(BidirectionalState *state, PGconn *n3_conn, + const char *prov_connstr, const char *source_node_name, + const char *source_sub_name, const char *source_slot_name, + const char *subscriber_name, const char *dbname, + const char *base_prov_connstr, + int stall_timeout, int max_wait); +static void clear_forwarding(PGconn *n3_conn, const char *source_dsn, + const char *source_sub_name, const char *source_slot_name, + int stall_timeout, int max_wait); +static void wait_for_sub_replicating(PGconn *conn, const char *sub_name, + int stall_timeout, int max_wait); +static void enable_peer_subs(PGconn *n3_conn, PeerNodeInfo *peers, int num_peers, + int stall_timeout, int max_wait); +static void create_subscription_on_conn(PGconn *conn, const char *sub_name, + const char *provider_dsn, const char *repsets, + const char *conn_label); +static void create_reverse_subscriptions(BidirectionalState *state, const char *subscriber_name, + const char *n3_dsn, const char *replication_sets, + const char *source_dsn, const char *source_node_name, + const char *dbname, const char *base_prov_connstr); +static void wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, int stall_timeout, int max_wait); +static void verify_dataflow_from_n3(PGconn *remote_conn, const char *remote_node_name, + const char *subscriber_name, const char *n3_marker, + int stall_timeout, int max_wait); +static void verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, + const char *remote_marker, const char *slot_name, + const char *sub_name, int stall_timeout, int max_wait); +static void verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *source_sub_name, const char *subscriber_name, + int stall_timeout, int max_wait); +static void set_spock_readonly(PGconn *conn, const char *value); +static Oid get_local_node_id(PGconn *conn); + +static PGconn * +connectdb(const char *connstr) +{ + PGconn *conn; + + conn = PQconnectdb(connstr); + if (PQstatus(conn) != CONNECTION_OK) + die(_("Connection to database failed: %s, connection string was: %s\n"), PQerrorMessage(conn), connstr); + + return conn; +} + +void +signal_handler(int sig) +{ + if (sig == SIGINT || sig == SIGTERM) + { + die(_("\nCanceling...\n")); + } +} + +/* + * Append str to buf with JSON string escaping applied, without the + * surrounding quotes (the caller supplies those). Control characters + * below 0x20 are emitted as \uXXXX. jsonapi.h provides a JSON parser but + * no encoder, so this is a small local encoder in the same style as + * src/bin/pg_combinebackup/write_manifest.c. + */ +static void +append_json_string(PQExpBuffer buf, const char *str) +{ + const char *p; + + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': + appendPQExpBufferStr(buf, "\\b"); + break; + case '\f': + appendPQExpBufferStr(buf, "\\f"); + break; + case '\n': + appendPQExpBufferStr(buf, "\\n"); + break; + case '\r': + appendPQExpBufferStr(buf, "\\r"); + break; + case '\t': + appendPQExpBufferStr(buf, "\\t"); + break; + case '"': + appendPQExpBufferStr(buf, "\\\""); + break; + case '\\': + appendPQExpBufferStr(buf, "\\\\"); + break; + default: + if ((unsigned char) *p < 0x20) + appendPQExpBuffer(buf, "\\u%04x", (unsigned char) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } +} + +/* + * Query the source for all peer nodes in the multi-master cluster. + * Returns the peer count; *peers_out is set to a pg_malloc0'd array. For + * each peer, sub_name is derived as "sub__" + * and slot_name is obtained via spock.spock_gen_slot_name() on the source. + */ +static int +discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out) +{ + static const char *discover_sql = + "SELECT DISTINCT n.node_name, ni.if_dsn" + " FROM spock.subscription s" + " JOIN spock.node n ON s.sub_origin = n.node_id" + " JOIN spock.node_interface ni ON ni.if_id = s.sub_origin_if" + " WHERE n.node_name != $1" + " ORDER BY n.node_name"; + const char *paramValues[3]; + PGresult *res; + PGresult *slot_res; + int npeers; + PeerNodeInfo *peers; + int i; + + paramValues[0] = source_node_name; + print_msg(VERBOSITY_DEBUG, _(" > %s [$1=%s]\n"), discover_sql, source_node_name); + res = PQexecParams(source_conn, discover_sql, + 1, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not discover peer nodes: %s"), + PQerrorMessage(source_conn)); + + npeers = PQntuples(res); + if (npeers == 0) + { + PQclear(res); + die(_("no peer nodes found; source does not appear to be part of a " + "multi-master cluster")); + } + + peers = pg_malloc0(npeers * sizeof(PeerNodeInfo)); + + for (i = 0; i < npeers; i++) + { + peers[i].node_name = pg_strdup(PQgetvalue(res, i, 0)); + peers[i].dsn = pg_strdup(PQgetvalue(res, i, 1)); + peers[i].sub_name = sub_name_for(subscriber_name, peers[i].node_name); + + paramValues[0] = dbname; + paramValues[1] = peers[i].node_name; + paramValues[2] = peers[i].sub_name; + print_msg(VERBOSITY_DEBUG, + _(" > SELECT spock.spock_gen_slot_name($1::name, $2::name, " + "$3::name) [$1=%s, $2=%s, $3=%s]\n"), + dbname, peers[i].node_name, peers[i].sub_name); + slot_res = PQexecParams(source_conn, + "SELECT spock.spock_gen_slot_name" + "($1::name, $2::name, $3::name)", + 3, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(slot_res) != PGRES_TUPLES_OK) + die(_("could not generate slot name for peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(source_conn)); + + peers[i].slot_name = pg_strdup(PQgetvalue(slot_res, 0, 0)); + PQclear(slot_res); + + print_msg(VERBOSITY_VERBOSE, + _(" discovered peer: %s (slot: %s)\n"), + peers[i].node_name, peers[i].slot_name); + } + + PQclear(res); + *peers_out = peers; + return npeers; +} + +/* + * Verify Spock version on conn (the source or a peer): an old apply + * worker would advance the wrong-named origin, so this must be checked + * everywhere up front, not just on the source. + */ +static void +check_spock_version_at_least_6(PGconn *conn, const char *node_label) +{ + PGresult *res; + + res = debug_exec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not query Spock extension version on \"%s\": %s"), + node_label, PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("Spock extension is not installed on \"%s\""), node_label); + } + { + const char *ver = PQgetvalue(res, 0, 0); + int major = 0; + + /* + * die() exits immediately -- ver points inside res, so it must not be + * PQclear()'d first (that would be a use-after-free when die()'s own + * formatting reads ver). + */ + if (sscanf(ver, "%d.", &major) < 1) + die(_("could not parse Spock version \"%s\" on \"%s\""), ver, node_label); + if (major < 6) + die(_("Spock version %s on \"%s\" is too old for bidirectional " + "join; require >= 6.0.0"), ver, node_label); + } + PQclear(res); +} + +/* + * Validate the actual directed subscription graph from one node's own + * catalog, not just a count: exactly one healthy (status = 'replicating') + * subscription from every other node in the set, no self-reference, no + * edge from outside the set, and no duplicate edge from the same origin + * regardless of status. + */ +static void +check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes) +{ + PGresult *res; + bool *healthy; + int *edge_count; + int i; + + /* + * spock.sub_show_status() (not raw sub_enabled) so "enabled" also means + * "actually replicating" -- a worker that's down or still initializing + * must not satisfy the mesh. + */ + res = debug_exec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check subscription topology on \"%s\": %s"), + this_node_name, PQerrorMessage(conn)); + } + + healthy = pg_malloc0(total_nodes * sizeof(bool)); + edge_count = pg_malloc0(total_nodes * sizeof(int)); + + for (i = 0; i < PQntuples(res); i++) + { + const char *origin_name = PQgetvalue(res, i, 0); + const char *status = PQgetvalue(res, i, 1); + int idx = -1; + int k; + + if (strcmp(origin_name, this_node_name) == 0) + die(_("node \"%s\" has a subscription whose origin is itself; " + "corrupt or misconfigured topology"), this_node_name); + + for (k = 0; k < total_nodes; k++) + { + if (strcmp(all_names[k], origin_name) == 0) + { + idx = k; + break; + } + } + if (idx == -1) + die(_("node \"%s\" has a subscription from \"%s\", which is not " + "part of the discovered node set; partial-mesh or " + "unknown-node topologies are not supported"), + this_node_name, origin_name); + + /* + * Count regardless of status: an extra disabled duplicate from the + * same origin is still a duplicate edge. + */ + edge_count[idx]++; + if (edge_count[idx] > 1) + die(_("node \"%s\" has more than one subscription from \"%s\" " + "(status \"%s\"); duplicate edges are not supported"), + this_node_name, origin_name, status); + + if (strcmp(status, "replicating") == 0) + healthy[idx] = true; + } + PQclear(res); + + for (i = 0; i < total_nodes; i++) + { + if (strcmp(all_names[i], this_node_name) == 0) + continue; /* skip self */ + if (!healthy[i]) + { + pg_free(healthy); + pg_free(edge_count); + die(_("node \"%s\" has no healthy (status = 'replicating') " + "subscription from \"%s\"; full-mesh topology of live " + "replication is required for bidirectional join"), + this_node_name, all_names[i]); + } + } + pg_free(healthy); + pg_free(edge_count); +} + +/* + * Confirm the peer identifies itself as the name it was discovered + * under, so a node-name collision or wrong DSN can't silently validate + * the mesh against the wrong node. + */ +static void +check_peer_identity(PGconn *peer_conn, const char *expected_name) +{ + PGresult *res; + + res = debug_exec(peer_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify identity of peer \"%s\": %s"), + expected_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), expected_name) != 0) + { + char *actual_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("peer discovered as \"%s\" identifies itself as \"%s\" once " + "connected; node-name/identity mismatch, refusing to trust " + "this topology\n"), expected_name, actual_name); + } + PQclear(res); +} + +/* + * Build a canonical, comparable fingerprint of one replication set as + * defined by its owning node: operation flags, each member table (sorted, + * with column list, row filter, and schema), then each member sequence + * (sorted). Scoped by node_id since spock.replication_set is keyed + * UNIQUE(set_nodeid, set_name) -- a set replicated via DDL becomes the + * replaying node's own row, not an echo. selected_filter restricts this + * to sets actually referenced by a subscription's sub_replication_sets, + * since unused/scratch repsets can legitimately differ between nodes. + */ +typedef struct RepsetFingerprintEntry +{ + char *set_name; + char *fingerprint; +} RepsetFingerprintEntry; + +static void +compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filter, + RepsetFingerprintEntry **out, int *nout) +{ + PGresult *res; + RepsetFingerprintEntry *entries; + int n; + int i; + PQExpBuffer query = createPQExpBuffer(); + + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_nodeid = %u" + " AND (%s)" + " ORDER BY set_name", node_id, selected_filter); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + destroyPQExpBuffer(query); + die(_("could not fingerprint replication sets: %s\n"), PQerrorMessage(conn)); + } + + n = PQntuples(res); + entries = pg_malloc0(n * sizeof(RepsetFingerprintEntry)); + + for (i = 0; i < n; i++) + { + PQExpBuffer fp = createPQExpBuffer(); + PGresult *tres; + PGresult *sres; + int j; + + entries[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + appendPQExpBuffer(fp, "flags=%s%s%s%s;", + PQgetvalue(res, i, 1)[0] == 't' ? "i" : "", + PQgetvalue(res, i, 2)[0] == 't' ? "u" : "", + PQgetvalue(res, i, 3)[0] == 't' ? "d" : "", + PQgetvalue(res, i, 4)[0] == 't' ? "t" : ""); + + printfPQExpBuffer(query, + "SELECT rts.set_reloid::regclass::text, rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rts.set_reloid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + tres = debug_exec(conn, query->data); + if (PQresultStatus(tres) != PGRES_TUPLES_OK) + { + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint table memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + + for (j = 0; j < PQntuples(tres); j++) + { + const char *qualified_table = PQgetvalue(tres, j, 0); + PGresult *cres; + PQExpBuffer schema_query = createPQExpBuffer(); + int k; + + appendPQExpBuffer(fp, "tbl=%s|cols=%s|filter=%s|schema=(", + qualified_table, + PQgetisnull(tres, j, 1) ? "*" : PQgetvalue(tres, j, 1), + PQgetisnull(tres, j, 2) ? "-" : PQgetvalue(tres, j, 2)); + + /* + * Schema fingerprint: relation kind and replica identity, then + * per-column name, type, typmod (varchar(10) vs varchar(100) is + * otherwise invisible), collation, nullability, and + * generated/identity status -- so a divergent column or relation + * definition is caught even if repset membership itself matches. + */ + printfPQExpBuffer(schema_query, + "SELECT relkind::text, relreplident::text" + " FROM pg_class WHERE oid = %s::regclass", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = debug_exec(conn, schema_query->data); + if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(schema_query); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint relation kind of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + appendPQExpBuffer(fp, "relkind=%s|replident=%s|", + PQgetvalue(cres, 0, 0), PQgetvalue(cres, 0, 1)); + PQclear(cres); + + printfPQExpBuffer(schema_query, + "SELECT a.attname, a.atttypid::regtype::text, a.atttypmod," + " a.attnotnull, a.attidentity, a.attgenerated," + " COALESCE(co.collname, '')" + " FROM pg_attribute a" + " LEFT JOIN pg_collation co ON co.oid = a.attcollation" + " WHERE a.attrelid = %s::regclass AND a.attnum > 0" + " AND NOT a.attisdropped ORDER BY a.attnum", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = debug_exec(conn, schema_query->data); + destroyPQExpBuffer(schema_query); + if (PQresultStatus(cres) != PGRES_TUPLES_OK) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint schema of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + for (k = 0; k < PQntuples(cres); k++) + appendPQExpBuffer(fp, "%s%s:%s:%s:notnull=%s:ident=%s:gen=%s:coll=%s", + k > 0 ? "," : "", + PQgetvalue(cres, k, 0), + PQgetvalue(cres, k, 1), + PQgetvalue(cres, k, 2), + PQgetvalue(cres, k, 3), + PQgetvalue(cres, k, 4), + PQgetvalue(cres, k, 5), + PQgetvalue(cres, k, 6)); + appendPQExpBufferStr(fp, ");"); + PQclear(cres); + } + PQclear(tres); + + printfPQExpBuffer(query, + "SELECT rss.set_seqoid::regclass::text" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rss.set_seqoid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + sres = debug_exec(conn, query->data); + if (PQresultStatus(sres) != PGRES_TUPLES_OK) + { + PQclear(sres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint sequence memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + for (j = 0; j < PQntuples(sres); j++) + appendPQExpBuffer(fp, "seq=%s;", PQgetvalue(sres, j, 0)); + PQclear(sres); + + entries[i].fingerprint = pg_strdup(fp->data); + destroyPQExpBuffer(fp); + } + PQclear(res); + destroyPQExpBuffer(query); + + *out = entries; + *nout = n; +} + +static void +free_repset_fingerprints(RepsetFingerprintEntry *entries, int n) +{ + int i; + + for (i = 0; i < n; i++) + { + pg_free(entries[i].set_name); + pg_free(entries[i].fingerprint); + } + pg_free(entries); +} + +/* + * Return a comma-separated list of every replication set actually + * referenced by conn's own subscriptions (sub_replication_sets), rather + * than every set that happens to exist locally. A --bidirectional join + * uses this to make the joining node inherit the sets already in use by + * the cluster it's joining, instead of accepting a separately specified + * list that could diverge from what check_replication_set_equivalence() + * (just below, via the identical query) validates. Caller frees the + * result. + */ +static char * +get_source_mesh_replication_sets(PGconn *conn) +{ + PGresult *res; + PQExpBuffer list; + char *result; + int i; + + res = debug_exec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine the cluster's replication sets: %s\n"), + PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "determine which replication sets to use\n")); + } + + list = createPQExpBuffer(); + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "%s%s", i > 0 ? "," : "", PQgetvalue(res, i, 0)); + PQclear(res); + + result = pg_strdup(list->data); + destroyPQExpBuffer(list); + return result; +} + +/* + * Build a SQL boolean expression ("set_name IN (...)") over the union of + * every replication set actually referenced by conn's own subscriptions + * (sub_replication_sets), rather than every set that happens to exist + * locally. Caller frees the result. + */ +static char * +build_selected_set_name_filter(PGconn *conn) +{ + PGresult *res; + PQExpBuffer filter; + char *result; + int i; + + res = debug_exec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine selected replication sets: %s\n"), PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "verify replication-set equivalence\n")); + } + + filter = createPQExpBuffer(); + appendPQExpBufferStr(filter, "set_name IN ("); + for (i = 0; i < PQntuples(res); i++) + { + char *name = PQgetvalue(res, i, 0); + + appendPQExpBuffer(filter, "%s%s", i > 0 ? ", " : "", + PQescapeLiteral(conn, name, strlen(name))); + } + appendPQExpBufferStr(filter, ")"); + PQclear(res); + + result = pg_strdup(filter->data); + destroyPQExpBuffer(filter); + return result; +} + +/* + * The forwarding path (peer -> source -> n3) and the future direct path + * (peer -> n3) must select exactly the same changes, or a change omitted + * on one path is lost once the direct subscription takes over. Compare + * every selected replication set's fingerprint between the source and + * each peer; reject any mismatch or missing/extra set on either side. + */ +static void +check_replication_set_equivalence(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +{ + Oid source_nodeid = get_local_node_id(source_conn); + char *selected_filter = build_selected_set_name_filter(source_conn); + RepsetFingerprintEntry *source_fps; + int num_source_fps; + int i; + + compute_repset_fingerprints(source_conn, source_nodeid, selected_filter, + &source_fps, &num_source_fps); + + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + Oid peer_nodeid; + RepsetFingerprintEntry *peer_fps; + int num_peer_fps; + int j; + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + peer_nodeid = get_local_node_id(peer_conn); + compute_repset_fingerprints(peer_conn, peer_nodeid, selected_filter, + &peer_fps, &num_peer_fps); + + /* + * die() exits immediately, so none of the branches below free + * source_fps/peer_fps before calling it -- freeing first and then + * still reading source_fps[j]/peer_fps[j] in the same die() call's + * arguments would be a use-after-free (the process is about to exit + * anyway; nothing else in this file frees before die() either). + */ + for (j = 0; j < num_source_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_peer_fps; k++) + { + if (strcmp(source_fps[j].set_name, peer_fps[k].set_name) != 0) + continue; + found = true; + if (strcmp(source_fps[j].fingerprint, peer_fps[k].fingerprint) != 0) + die(_("replication set \"%s\" differs between the source " + "and peer \"%s\" (membership, flags, columns, row " + "filter, or schema) -- the forwarding path and a " + "future direct peer subscription would not select " + "the same changes, risking permanently lost data " + "on cutover. Reconcile the definitions before " + "retrying.\n"), + source_fps[j].set_name, peers[i].node_name); + break; + } + if (!found) + die(_("replication set \"%s\" exists on the source but not " + "on peer \"%s\"\n"), source_fps[j].set_name, peers[i].node_name); + } + for (j = 0; j < num_peer_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_source_fps; k++) + if (strcmp(peer_fps[j].set_name, source_fps[k].set_name) == 0) + { + found = true; + break; + } + if (!found) + die(_("replication set \"%s\" exists on peer \"%s\" but not " + "on the source\n"), peer_fps[j].set_name, peers[i].node_name); + } + + free_repset_fingerprints(peer_fps, num_peer_fps); + PQfinish(peer_conn); + } + + free_repset_fingerprints(source_fps, num_source_fps); + pg_free(selected_filter); +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0 on every node, track_commit_timestamp + * on, no pending DDL, an actual full-mesh subscription graph (not just a + * count), replication-set/schema equivalence across the source and every + * peer, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + int total_nodes = num_peers + 1; + char **all_names = pg_malloc(total_nodes * sizeof(char *)); + + all_names[0] = pg_strdup(source_node_name); + for (i = 0; i < num_peers; i++) + all_names[i + 1] = pg_strdup(peers[i].node_name); + + check_spock_version_at_least_6(source_conn, "source"); + + /* track_commit_timestamp must be on at the source */ + res = debug_exec(source_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check track_commit_timestamp: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + die(_("track_commit_timestamp must be on for bidirectional join (source)")); + PQclear(res); + + /* + * All outbound replication caught up to the source's current WAL position + * -- i.e. nothing (DDL or data) still in flight to an existing peer. + * spock.queue's row count is not a usable signal here: queue_message() + * (spock_queue.c) only ever inserts into it, so its count is + * monotonically non-decreasing and is never zero on any node that has + * replicated so much as a single DDL statement. + */ + res = debug_exec(source_conn, + "SELECT COUNT(*) FROM pg_replication_slots" + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" + " AND (confirmed_flush_lsn IS NULL" + " OR confirmed_flush_lsn < pg_current_wal_lsn())"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check replication slot lag: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + die(_("source has unreplicated changes pending to an existing peer; " + "wait for replication to drain before joining")); + PQclear(res); + + /* Full-mesh directed-graph check, from the source's own perspective. */ + check_mesh_edges(source_conn, source_node_name, all_names, total_nodes); + + /* + * Per-peer: connectivity, Spock version, track_commit_timestamp, and the + * full-mesh directed-graph check from each peer's own perspective (a mesh + * that's only complete as seen from the source is not a mesh). + */ + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + + print_msg(VERBOSITY_VERBOSE, + _(" checking peer %s ...\n"), peers[i].node_name); + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + check_peer_identity(peer_conn, peers[i].node_name); + check_spock_version_at_least_6(peer_conn, peers[i].node_name); + + res = debug_exec(peer_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + /* + * die() exits immediately -- PQerrorMessage() needs peer_conn + * still open, so PQfinish() must not run first (that would be a + * use-after-free when die()'s own formatting reads it). + */ + PQclear(res); + die(_("could not check track_commit_timestamp on peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + { + PQclear(res); + PQfinish(peer_conn); + die(_("track_commit_timestamp must be on for bidirectional join " + "(peer \"%s\")"), peers[i].node_name); + } + PQclear(res); + + check_mesh_edges(peer_conn, peers[i].node_name, all_names, total_nodes); + + PQfinish(peer_conn); + } + + /* Replication-set & schema equivalence: run once the mesh is sound. */ + check_replication_set_equivalence(source_conn, peers, num_peers); + + for (i = 0; i < total_nodes; i++) + pg_free(all_names[i]); + pg_free(all_names); + + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); +} + +/* + * The physical-backup path runs once per data directory, so it requires + * exactly one spock-configured database on the source instance. Checked + * against actual spock configuration, not --databases/--provider-dsn, + * since the instance can host other unrelated databases. Fails closed: + * any database we cannot inspect aborts the run rather than being + * treated as spock-free. datallowconn is not used to skip databases -- + * a database with connections disabled can still hold spock catalog + * state -- only true templates are excluded. + */ +static void +check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname) +{ + PGresult *res; + int i; + PQExpBuffer others = createPQExpBuffer(); + int other_count = 0; + + res = debug_exec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not list databases on source: %s\n"), PQerrorMessage(conn)); + } + + for (i = 0; i < PQntuples(res); i++) + { + char *dbname = PQgetvalue(res, i, 0); + char *db_connstr; + PGconn *db_conn; + PGresult *ext_res; + PGresult *node_res; + + if (strcmp(dbname, current_dbname) == 0) + continue; + + db_connstr = get_connstr((char *) base_prov_connstr, dbname); + db_conn = PQconnectdb(db_connstr); + if (PQstatus(db_conn) != CONNECTION_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not connect to " + "\"%s\" to check: %s\n"), dbname, errmsg); + } + + ext_res = debug_exec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not query \"%s\": " + "%s\n"), dbname, errmsg); + } + + if (PQntuples(ext_res) > 0) + { + node_res = debug_exec(db_conn, "SELECT 1 FROM spock.local_node"); + if (PQresultStatus(node_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(node_res); + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on " + "the source has spock configured, but could not query " + "spock.local_node in \"%s\": %s\n"), dbname, errmsg); + } + + if (PQntuples(node_res) > 0) + { + appendPQExpBuffer(others, "%s%s", other_count ? ", " : "", dbname); + other_count++; + } + PQclear(node_res); + } + PQclear(ext_res); + PQfinish(db_conn); + } + PQclear(res); + + if (other_count > 0) + die(_("--bidirectional requires exactly one spock-configured database " + "on the source instance; also found spock configured on: %s\n"), + others->data); + + destroyPQExpBuffer(others); +} + +/* + * A physical base backup copies native (non-spock) logical subscriptions + * too, which DROP EXTENSION spock doesn't touch. Once n3 is promoted and + * restarted, an enabled native subscription would start consuming from + * its provider as a second, unintended consumer. pg_subscription is a + * shared catalog, so one query sees every database's rows. + */ +static void +check_no_native_subscriptions(PGconn *conn) +{ + PGresult *res; + + res = debug_exec(conn, + "SELECT s.subname, d.datname" + " FROM pg_subscription s" + " JOIN pg_database d ON d.oid = s.subdbid" + " WHERE s.subenabled"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check for native logical subscriptions on the " + "source instance: %s\n"), PQerrorMessage(conn)); + } + + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; + + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s (database %s)", + PQgetvalue(res, i, 0), PQgetvalue(res, i, 1)); + + PQclear(res); + die(_("--bidirectional requires no enabled native (non-spock) logical " + "subscriptions anywhere on the source instance -- a physical " + "backup would copy them, and they would start consuming on " + "the new node as an unintended second consumer once " + "promoted: %s\nDisable or drop these subscriptions before " + "retrying.\n"), list->data); + } + PQclear(res); +} + +/* + * Manifest field names, shared between serialize_manifest() and the + * manifest_* parser callbacks below, so a typo in one becomes a + * compile error instead of a silent read/write mismatch. + */ +#define MF_SUBSCRIBER_NAME "subscriber_name" +#define MF_DBNAME "dbname" +#define MF_SOURCE_DSN "source_dsn" +#define MF_SOURCE_SLOT_NAME "source_slot_name" +#define MF_SOURCE_RESTORE_LSN "source_restore_lsn" +#define MF_NODE_DSN "node_dsn" +#define MF_NODE_SYSID "node_sysid" +#define MF_SOURCE_REVERSE_SUB_CREATED "source_reverse_sub_created" +#define MF_PEERS "peers" +#define MF_NODE_NAME "node_name" +#define MF_PEER_DSN "peer_dsn" +#define MF_SUB_NAME_ON_N3 "sub_name_on_n3" +#define MF_PEER_SLOT_NAME "peer_slot_name" +#define MF_SLOT_CREATION_LSN "slot_creation_lsn" +#define MF_REVERSE_SUB_CREATED "reverse_sub_created" + +/* + * Append one """: """ field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). A NULL value serializes as an empty string, matching + * append_json_string()'s existing null-becomes-empty convention here. + */ +static void +append_json_str_field(PQExpBuffer buf, const char *indent, const char *field_name, + const char *value, bool trailing_comma) +{ + appendPQExpBuffer(buf, "%s\"%s\": \"", indent, field_name); + if (value) + append_json_string(buf, value); + appendPQExpBufferStr(buf, trailing_comma ? "\",\n" : "\"\n"); +} + +/* + * Append one """: true|false" field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). + */ +static void +append_json_bool_field(PQExpBuffer buf, const char *indent, const char *field_name, + bool value, bool trailing_comma) +{ + appendPQExpBuffer(buf, "%s\"%s\": %s%s\n", indent, field_name, + value ? "true" : "false", trailing_comma ? "," : ""); +} + +/* + * Hand-serialize the bidirectional state manifest schema into buf, as + * JSON, with string values escaped by append_json_string(). + */ +static void +serialize_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, PQExpBuffer buf) +{ + int i; + + appendPQExpBufferStr(buf, "{\n"); + appendPQExpBufferStr(buf, " \"version\": 1,\n"); + + append_json_str_field(buf, " ", MF_SUBSCRIBER_NAME, subscriber_name, true); + append_json_str_field(buf, " ", MF_DBNAME, dbname, true); + append_json_str_field(buf, " ", MF_SOURCE_DSN, source_dsn, true); + append_json_str_field(buf, " ", MF_SOURCE_SLOT_NAME, state->source_slot_name, true); + append_json_str_field(buf, " ", MF_SOURCE_RESTORE_LSN, state->source_restore_lsn, true); + append_json_str_field(buf, " ", MF_NODE_DSN, state->node_dsn, true); + append_json_str_field(buf, " ", MF_NODE_SYSID, state->node_sysid, true); + append_json_bool_field(buf, " ", MF_SOURCE_REVERSE_SUB_CREATED, + state->source_reverse_sub_created, true); + + appendPQExpBuffer(buf, " \"%s\": [\n", MF_PEERS); + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *p = &state->peers[i]; + bool last = (i == state->num_peers - 1); + + appendPQExpBufferStr(buf, " {\n"); + append_json_str_field(buf, " ", MF_NODE_NAME, p->node_name, true); + append_json_str_field(buf, " ", MF_PEER_DSN, p->dsn, true); + append_json_str_field(buf, " ", MF_SUB_NAME_ON_N3, p->sub_name, true); + append_json_str_field(buf, " ", MF_PEER_SLOT_NAME, p->slot_name, true); + append_json_str_field(buf, " ", MF_SLOT_CREATION_LSN, p->slot_creation_lsn, true); + append_json_bool_field(buf, " ", MF_REVERSE_SUB_CREATED, + p->reverse_sub_created, false); + appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); + } + appendPQExpBufferStr(buf, " ]\n"); + appendPQExpBufferStr(buf, "}\n"); +} + +/* + * Atomically replace path's contents with the len bytes at data: write + * to a ".tmp" sibling with mode 0600 up front (never a post-hoc + * chmod, since the caller's data can embed a password), fsync, rename + * over path, then fsync the containing directory -- a crash right + * after this returns must not lose the write (e.g. the only cleanup + * record for a resource created just before it). O_EXCL|O_NOFOLLOW + * refuses to write through a pre-existing file or planted symlink, + * except a leftover .tmp from a previous crashed run. + */ +static void +durably_replace_manifest(const char *path, const char *data, size_t len) +{ + char tmp_path[MAXPGPATH]; + int fd; + ssize_t written; + + snprintf(tmp_path, MAXPGPATH, "%s.tmp", path); + + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + if (fd < 0 && errno == EEXIST) + { + if (unlink(tmp_path) != 0) + die(_("could not remove stale manifest temp file \"%s\": %s"), + tmp_path, strerror(errno)); + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + } + if (fd < 0) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + written = write(fd, data, len); + if (written < 0 || (size_t) written != len) + { + close(fd); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + + if (fsync(fd) != 0) + { + close(fd); + unlink(tmp_path); + die(_("could not fsync manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (close(fd) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + path, strerror(errno)); + + /* + * fsync_parent_path() already treats "filesystem doesn't support + * directory fsync" as success internally, so a nonzero return here is a + * genuine failure that can orphan the resource(s) this manifest is the + * only record of, after a crash. Fatal, like the durability steps above. + */ + if (fsync_parent_path(path) != 0) + die(_("could not fsync directory containing \"%s\": %s\n"), + path, strerror(errno)); +} + +/* + * Write the bidirectional state manifest to state->manifest_path. + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + + serialize_manifest(state, subscriber_name, dbname, source_dsn, buf); + durably_replace_manifest(state->manifest_path, buf->data, buf->len); + + destroyPQExpBuffer(buf); +} + +/* + * PG16 and earlier declare the JsonSemAction callbacks as returning void; + * PG16 was actually already JsonParseErrorType-returning for the callbacks + * we use here, but PG15 is not, so this is the one boundary that matters + * for the manifest_* functions below. + */ +#if PG_VERSION_NUM >= 160000 +#define JSON_CALLBACK_RESULT JsonParseErrorType +#define JSON_CALLBACK_RETURN return JSON_SUCCESS +#else +#define JSON_CALLBACK_RESULT void +#define JSON_CALLBACK_RETURN return +#endif + +/* + * Semantic-action state for read_manifest(). Passed as void *semstate to all + * pg_parse_json callbacks; tracks nesting depth and accumulates field values. + */ +typedef struct ManifestParseState +{ + /* outputs written by scalar callback */ + char **subscriber_name_out; + char **dbname_out; + char **source_dsn_out; + BidirectionalState *bidir; + + /* parser context */ + int depth; /* object/array nesting depth */ + bool in_peers; /* inside the top-level "peers" array */ + bool in_peer_obj; /* inside one peer object */ + char *cur_field; /* current object field name (owned by us) */ + + /* per-peer accumulator, flushed on each object_end inside peers */ + char *peer_node_name; + char *peer_dsn; + char *peer_sub_name; + char *peer_slot_name; + char *peer_slot_creation_lsn; + bool peer_reverse_sub_created; + int peer_capacity; +} ManifestParseState; + +static JSON_CALLBACK_RESULT +manifest_object_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->in_peers && s->depth == 3) + s->in_peer_obj = true; + JSON_CALLBACK_RETURN; +} + +static JSON_CALLBACK_RESULT +manifest_object_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peer_obj && s->depth == 3) + { + int i = s->bidir->num_peers; + + if (i >= s->peer_capacity) + { + s->peer_capacity = (s->peer_capacity > 0) ? s->peer_capacity * 2 : 4; + s->bidir->peers = pg_realloc(s->bidir->peers, + s->peer_capacity * sizeof(PeerNodeInfo)); + } + s->bidir->peers[i].node_name = s->peer_node_name; + s->bidir->peers[i].dsn = s->peer_dsn; + s->bidir->peers[i].sub_name = s->peer_sub_name; + s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->peers[i].slot_creation_lsn = s->peer_slot_creation_lsn; + s->bidir->peers[i].reverse_sub_created = s->peer_reverse_sub_created; + s->bidir->num_peers++; + s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->peer_slot_creation_lsn = NULL; + s->peer_reverse_sub_created = false; + s->in_peer_obj = false; + } + s->depth--; + JSON_CALLBACK_RETURN; +} + +static JSON_CALLBACK_RESULT +manifest_array_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->depth == 2 && s->cur_field != NULL && + strcmp(s->cur_field, MF_PEERS) == 0) + s->in_peers = true; + JSON_CALLBACK_RETURN; +} + +static JSON_CALLBACK_RESULT +manifest_array_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peers && s->depth == 2) + s->in_peers = false; + s->depth--; + JSON_CALLBACK_RETURN; +} + +static JSON_CALLBACK_RESULT +manifest_ofield_start(void *st, char *fname, bool isnull) +{ + ManifestParseState *s = (ManifestParseState *) st; + + (void) isnull; + pg_free(s->cur_field); + s->cur_field = pg_strdup(fname); + pg_free(fname); /* callback owns the token */ + JSON_CALLBACK_RETURN; +} + +static JSON_CALLBACK_RESULT +manifest_scalar(void *st, char *token, JsonTokenType tokentype) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->cur_field == NULL) + { + pg_free(token); + JSON_CALLBACK_RETURN; + } + + /* + * Per-peer creation-state flags are JSON booleans, not strings -- handle + * them before the string-only fields below (which free and ignore + * anything that isn't JSON_TOKEN_STRING). + */ + if (s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, MF_REVERSE_SUB_CREATED) == 0) + s->peer_reverse_sub_created = value; + pg_free(token); + JSON_CALLBACK_RETURN; + } + + /* Top-level creation-state flags are also JSON booleans. */ + if (!s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, MF_SOURCE_REVERSE_SUB_CREATED) == 0) + s->bidir->source_reverse_sub_created = value; + pg_free(token); + JSON_CALLBACK_RETURN; + } + + if (tokentype != JSON_TOKEN_STRING) + { + pg_free(token); + JSON_CALLBACK_RETURN; + } + + if (!s->in_peer_obj) + { + /* top-level scalar fields */ + if (strcmp(s->cur_field, MF_SUBSCRIBER_NAME) == 0) + *s->subscriber_name_out = token; + else if (strcmp(s->cur_field, MF_DBNAME) == 0) + *s->dbname_out = token; + else if (strcmp(s->cur_field, MF_SOURCE_DSN) == 0) + *s->source_dsn_out = token; + else if (strcmp(s->cur_field, MF_SOURCE_SLOT_NAME) == 0) + s->bidir->source_slot_name = token; + else if (strcmp(s->cur_field, MF_NODE_DSN) == 0) + s->bidir->node_dsn = token; + else if (strcmp(s->cur_field, MF_NODE_SYSID) == 0) + s->bidir->node_sysid = token; + else + pg_free(token); + } + else + { + /* per-peer scalar fields */ + if (strcmp(s->cur_field, MF_NODE_NAME) == 0) + s->peer_node_name = token; + else if (strcmp(s->cur_field, MF_PEER_DSN) == 0) + s->peer_dsn = token; + else if (strcmp(s->cur_field, MF_SUB_NAME_ON_N3) == 0) + s->peer_sub_name = token; + else if (strcmp(s->cur_field, MF_PEER_SLOT_NAME) == 0) + s->peer_slot_name = token; + else if (strcmp(s->cur_field, MF_SLOT_CREATION_LSN) == 0) + s->peer_slot_creation_lsn = token; + else + pg_free(token); + } + JSON_CALLBACK_RETURN; +} + +/* + * Read the bidirectional manifest from manifest_path. Returns false if + * the file does not exist (nothing to clean up); dies if it exists but + * cannot be read or is malformed. On success, sets *subscriber_name_out, + * *dbname_out, *source_dsn_out, and populates state->peers[]. + * + * Uses pg_parse_json (common/jsonapi.h) for JSON lexing, so string + * quoting, escape sequences, and nesting are handled correctly. + */ +static bool +read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out) +{ + struct stat st; + char *content; + FILE *f; + JsonLexContext *lex; + JsonSemAction sem; + ManifestParseState pstate; + JsonParseErrorType result; + + if (stat(manifest_path, &st) != 0) + return false; + + content = pg_malloc(st.st_size + 1); + f = fopen(manifest_path, "r"); + if (f == NULL) + die(_("could not open manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + + if ((size_t) fread(content, 1, st.st_size, f) != (size_t) st.st_size) + { + fclose(f); + die(_("could not read manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + } + content[st.st_size] = '\0'; + fclose(f); + + memset(&pstate, 0, sizeof(pstate)); + pstate.subscriber_name_out = subscriber_name_out; + pstate.dbname_out = dbname_out; + pstate.source_dsn_out = source_dsn_out; + pstate.bidir = state; + + memset(&sem, 0, sizeof(sem)); + sem.semstate = &pstate; + sem.object_start = manifest_object_start; + sem.object_end = manifest_object_end; + sem.array_start = manifest_array_start; + sem.array_end = manifest_array_end; + sem.object_field_start = manifest_ofield_start; + sem.scalar = manifest_scalar; + + /* + * PG17 added a reusable JsonLexContext (leading lex argument) plus + * freeJsonLexContext() to release it; older majors allocate and manage + * the context internally and have no free function to call. + */ +#if PG_VERSION_NUM >= 170000 + lex = makeJsonLexContextCstringLen(NULL, content, st.st_size, + PG_UTF8, true); +#else + lex = makeJsonLexContextCstringLen(content, st.st_size, PG_UTF8, true); +#endif + result = pg_parse_json(lex, &sem); + pg_free(content); + pg_free(pstate.cur_field); + + if (result != JSON_SUCCESS) + { + /* + * json_errdetail() is FRONTEND-safe only from PG17 on; PG15/16 + * restrict it to the backend, so older majors get a plainer message + * naming the JsonParseErrorType instead. + */ +#if PG_VERSION_NUM >= 170000 + char *detail = json_errdetail(result, lex); + + freeJsonLexContext(lex); + die(_("manifest file \"%s\" is malformed: %s"), manifest_path, detail); +#else + die(_("manifest file \"%s\" is malformed (JSON parse error %d)"), + manifest_path, (int) result); +#endif + } +#if PG_VERSION_NUM >= 170000 + freeJsonLexContext(lex); +#endif + + if (!*subscriber_name_out || !*dbname_out || !*source_dsn_out) + die(_("manifest file \"%s\" is malformed or missing required fields"), + manifest_path); + + return true; +} + +/* + * If data_dir holds a running postmaster, stop it (fast mode) and wait + * for shutdown. No-op if data_dir is unset, doesn't exist, or has no + * postmaster.pid. + */ +static void +stop_postgres_in_data_dir(void) +{ + struct stat st; + + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s ...\n"), data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } +} + +/* + * If data_dir exists, remove it when force is true (stopping postgres in + * it first, defensively, in case the caller hasn't already); if force is + * false, leave it in place with a hint. Returns false only when removal + * was attempted and actually failed; a missing data_dir, an unset one, + * or force being false are all "nothing to report" and return true. + */ +static bool +remove_data_dir_if_forced(bool force) +{ + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return true; + + if (!force) + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + return true; + } + + stop_postgres_in_data_dir(); + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + return false; + } + + return true; +} + +/* + * Check whether conn's system_identifier (from pg_control_system()) + * matches expected_sysid. Any failure to confirm -- query error, no + * row, or an outright mismatch -- returns false. + */ +static bool +check_sysid_matches(PGconn *conn, const char *expected_sysid) +{ + PGresult *res; + bool matches; + + res = debug_exec(conn, "SELECT system_identifier FROM pg_control_system()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + return false; + } + matches = strcmp(PQgetvalue(res, 0, 0), expected_sysid) == 0; + PQclear(res); + return matches; +} + +/* + * Outcome of an idempotent drop-if-exists cleanup operation. A plain bool + * can't tell "nothing to do" apart from "confirmed gone", which callers + * that gate further action (or --cleanup's own retry decision) on actual + * removal need to distinguish from a query failure. + */ +typedef enum +{ + CLEANUP_DROP_FAILED, /* query failed; removal not confirmed */ + CLEANUP_DROP_ABSENT, /* query succeeded; resource did not exist */ + CLEANUP_DROP_REMOVED /* query succeeded; resource was dropped */ +} CleanupDropOutcome; + +/* + * Drop the replication slot named slot_name on conn if it exists. + * node_label identifies the node in progress/warning messages. + */ +static CleanupDropOutcome +drop_logical_slot_if_exists(PGconn *conn, const char *slot_name, const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + slot_name); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop slot %s on %s: %s\n"), + slot_name, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (PQntuples(res) > 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped slot %s on %s\n"), slot_name, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop the reverse subscription named "sub__" + * on conn, via spock.sub_drop(..., ifexists := true) -- an absent + * subscription is not an error. node_label identifies the node in + * progress/warning messages. spock.sub_drop() itself returns whether it + * found and dropped a subscription, which is read back here to distinguish + * CLEANUP_DROP_ABSENT from CLEANUP_DROP_REMOVED; CLEANUP_DROP_FAILED (and a + * warning) is returned only on an actual query failure. spock.sub_drop() + * is declared RETURNS oid in the catalog (sql/spock--6.0.0.sql) even + * though its C implementation (spock_drop_subscription) returns a bool + * Datum, so the wire text is "0"/"1", not "f"/"t" -- compare against "0" + * rather than checking for boolean-formatted text. + */ +static CleanupDropOutcome +drop_reverse_sub(PGconn *conn, const char *node_name, const char *subscriber_name, + const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *reverse_sub = sub_name_for(node_name, subscriber_name); + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(conn, reverse_sub, strlen(reverse_sub))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on %s: %s\n"), + reverse_sub, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped reverse subscription %s on %s\n"), + reverse_sub, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + pg_free(reverse_sub); + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop every subscription this run may have created on n3 itself: the + * catchup subscription to the source and any disabled peer subscriptions. + * A freshly-provisioned n3 has no other legitimate spock.subscription + * rows, so it's safe to drop everything found -- but only once node_sysid + * confirms node_dsn still reaches that same n3, since a manifest can + * outlive the node it describes (DNS change, load balancer, reused port). + * node_dsn is only set once node_create() has run, so its absence just + * means there's nothing on n3 yet, and this returns true without doing + * anything. spock.sub_drop() on n3 kills that subscription's local apply + * worker and drops the matching remote slot on its origin itself, so n3 + * never needs to be stopped just to release a slot it holds open + * elsewhere. + */ +static bool +cleanup_verified_subscriber_node(BidirectionalState *state, const char *subscriber_name) +{ + PGconn *n3_conn; + PGresult *res; + PQExpBuffer query; + int i; + bool fully_cleaned = true; + + if (!state->node_dsn || !state->node_dsn[0]) + return true; + + n3_conn = PQconnectdb(state->node_dsn); + if (PQstatus(n3_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to subscriber \"%s\"; its " + "subscription(s) may still exist: %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + PQfinish(n3_conn); + return false; + } + + if (!state->node_sysid || !state->node_sysid[0] || + !check_sysid_matches(n3_conn, state->node_sysid)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: node_dsn for subscriber \"%s\" cannot be " + "confirmed to still identify the node this run " + "created (missing or mismatched system identifier); " + "refusing to drop subscriptions there. Investigate " + "manually.\n"), subscriber_name); + PQfinish(n3_conn); + return false; + } + + query = createPQExpBuffer(); + res = debug_exec(n3_conn, "SELECT sub_name FROM spock.subscription"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + for (i = 0; i < PQntuples(res); i++) + { + char *sub_name = PQgetvalue(res, i, 0); + PGresult *drop_res; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(n3_conn, sub_name, strlen(sub_name))); + drop_res = debug_exec(n3_conn, query->data); + if (PQresultStatus(drop_res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_NORMAL, + _(" dropped subscriber subscription %s\n"), + sub_name); + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop subscriber " + "subscription %s: %s\n"), + sub_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(drop_res); + } + } + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not list subscriptions on " + "subscriber \"%s\": %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(res); + destroyPQExpBuffer(query); + PQfinish(n3_conn); + + return fully_cleaned; +} + +/* + * Drop resources this run may have created on the source and each peer: + * the source's replication slot and reverse subscription (if recorded as + * created), and each peer's replication slot and reverse subscription. + * Unlike n3 (a freshly-provisioned node with no unrelated subscriptions, + * see cleanup_verified_subscriber_node()), the source and peers have their + * own pre-existing state that must not be touched -- so every drop here is + * gated by a flag or an LSN this run itself recorded, and reverse + * subscriptions are targeted by name via drop_reverse_sub() rather than + * dropping everything found. + */ +static bool +cleanup_upstream_node_resources(BidirectionalState *state, const char *subscriber_name, + const char *source_dsn) +{ + PGconn *source_conn; + PGresult *res; + int i; + bool fully_cleaned = true; + + source_conn = PQconnectdb(source_dsn); + if (PQstatus(source_conn) != CONNECTION_OK) + { + if (state->source_slot_name && state->source_slot_name[0]) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; slot %s " + "may still exist: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } + PQfinish(source_conn); + source_conn = NULL; + } + + /* Drop source replication slot if it was created */ + if (source_conn && state->source_slot_name && state->source_slot_name[0]) + { + if (drop_logical_slot_if_exists(source_conn, state->source_slot_name, + "the source") == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + /* + * Drop the reverse subscription on the source if this run recorded having + * created it. Unlike n3 (whose block drops every subscription it finds, + * since a fresh n3 has no other legitimate ones), the source has its own + * pre-existing, unrelated subscriptions that must not be touched -- so + * this is gated by the flag and targets the specific reverse subscription + * by name. + */ + if (source_conn && state->source_reverse_sub_created) + { + res = debug_exec(source_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + print_msg(VERBOSITY_NORMAL, + _("warning: could not determine the source's node name; " + "its reverse subscription may still exist: %s\n"), + PQerrorMessage(source_conn)); + fully_cleaned = false; + } + else + { + char *source_node_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + if (drop_reverse_sub(source_conn, source_node_name, subscriber_name, + "the source") == CLEANUP_DROP_FAILED) + fully_cleaned = false; + pg_free(source_node_name); + } + } + + /* Per-peer: drop slot and any reverse subscription */ + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + + if (!peer->dsn || !peer->dsn[0]) + continue; + + /* + * Only attempt to drop -- and only require connectivity for -- + * resources this run actually recorded as created. Slot/sub names + * are deterministic, not per-run unique, so --cleanup must not touch + * a same-named resource from an unrelated join, nor report + * "incomplete" over a peer that was never touched. + */ + if ((!peer->slot_creation_lsn || !peer->slot_creation_lsn[0]) && + !peer->reverse_sub_created) + continue; + + peer_conn = PQconnectdb(peer->dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to peer \"%s\"; its slot/" + "subscription may still exist: %s\n"), + peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; + PQfinish(peer_conn); + continue; + } + + if (peer->slot_creation_lsn && peer->slot_creation_lsn[0] && + peer->slot_name && peer->slot_name[0]) + { + if (drop_logical_slot_if_exists(peer_conn, peer->slot_name, + peer->node_name) == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + /* + * Drop the reverse subscription (peer -> new subscriber) only if this + * run recorded having created it. The sub_drop second argument is + * ifexists=true, so an absent subscription is not an error -- only an + * actual query failure counts against fully_cleaned. + */ + if (peer->reverse_sub_created) + { + if (drop_reverse_sub(peer_conn, peer->node_name, subscriber_name, + peer->node_name) == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + PQfinish(peer_conn); + print_msg(VERBOSITY_NORMAL, + _(" cleaned up peer %s\n"), peer->node_name); + } + + if (source_conn) + PQfinish(source_conn); + + return fully_cleaned; +} + +/* + * Idempotently remove bidirectional join state from all reachable nodes: + * n3 itself, the source, and each peer, via + * cleanup_verified_subscriber_node() and cleanup_upstream_node_resources(). + * Connectivity and drop failures are logged as warnings, not fatal, so + * cleanup attempts every remaining resource -- but each failure is + * tracked, and the function returns true only if every recorded resource + * was confirmed gone. The manifest/sidecar record (the only way to + * retry) is removed only on a true return; an incomplete cleanup keeps + * it and the caller exits non-zero. + */ +static bool +cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir) +{ + bool fully_cleaned = true; + + print_msg(VERBOSITY_NORMAL, + _("Cleaning up partial bidirectional join state ...\n")); + + if (!cleanup_verified_subscriber_node(state, subscriber_name)) + fully_cleaned = false; + + if (!cleanup_upstream_node_resources(state, subscriber_name, source_dsn)) + fully_cleaned = false; + + /* + * Stop n3's postmaster unconditionally (not gated by --force, which only + * governs removing the data directory). check_data_dir() and + * check_reused_data_dir_is_safe() explicitly support resuming a join into + * this same data_dir after a failed attempt, and that resume path + * (main(), the "start -l ..." pg_ctl call before catchup) assumes + * postgres is not already running here; leaving it up after `--cleanup` + * would make the very next retry fail outright. Every drop above already + * ran while n3 was still reachable -- including the source's and each + * peer's reverse subscription, whose provider is n3, so dropping them + * needs a live connection back to it -- so this is just shutdown, not a + * substitute for them. + */ + stop_postgres_in_data_dir(); + + /* + * The data directory a partial run may have created via basebackup. Never + * touch it without --force. + */ + if (!remove_data_dir_if_forced(force_rm_datadir)) + fully_cleaned = false; + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: some resource(s) above could not be " + "confirmed removed. Keeping the manifest/sidecar record " + "so --cleanup can be retried.\n")); + return false; + } + + /* + * Every remote/local resource above was confirmed gone; now remove the + * retry record(s) themselves. An unexpected removal failure here + * (anything but ENOENT, i.e. already gone) must also flip fully_cleaned + * -- otherwise the caller reports success and exits 0 while a stale + * record that still references now-removed resources lingers on disk, + * which a later --cleanup could misread as current. + */ + if (state->manifest_path && state->manifest_path[0]) + { + if (unlink(state->manifest_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove manifest %s: %s\n"), + state->manifest_path, strerror(errno)); + fully_cleaned = false; + } + } + + /* + * Also remove any pending-cleanup sidecar, even if it wasn't the file + * that drove this cleanup: a stale one left behind by an earlier run + * whose own sidecar-unlink failed could otherwise be misread as current + * by a later --cleanup once the manifest above is gone, reporting + * resources as still-pending that were, in fact, already confirmed + * removed here. + */ + if (data_dir != NULL && data_dir[0]) + { + char sidecar_path[MAXPGPATH]; + + snprintf(sidecar_path, MAXPGPATH, "%s.spock_bidir_pending.json", data_dir); + if (unlink(sidecar_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed pending sidecar %s\n"), sidecar_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove pending sidecar %s: %s\n"), + sidecar_path, strerror(errno)); + fully_cleaned = false; + } + } + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: the manifest or sidecar record could " + "not be removed even though every resource it tracked " + "was confirmed gone. Retry --cleanup to remove the " + "stale record.\n")); + return false; + } + + print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); + return true; +} + + +/* + * Phase 1: parse command-line options into ctx, then validate them and + * derive the bidirectional manifest/sidecar paths. Handles --help + * directly (exits). Self-contained -- no phase runs before this one. + */ +static void +parse_options(int argc, char **argv, SubscriberCreateContext *ctx) +{ + int i; + int c; + int optindex; + + static struct option long_options[] = { + {"subscriber-name", required_argument, NULL, 'n'}, + {"pgdata", required_argument, NULL, 'D'}, + {"provider-dsn", required_argument, NULL, 1}, + {"subscriber-dsn", required_argument, NULL, 2}, + {"replication-sets", required_argument, NULL, 3}, + {"postgresql-conf", required_argument, NULL, 4}, + {"hba-conf", required_argument, NULL, 5}, + {"recovery-conf", required_argument, NULL, 6}, + {"stop", no_argument, NULL, 's'}, + {"drop-slot-if-exists", no_argument, NULL, 7}, + {"apply-delay", required_argument, NULL, 8}, + {"databases", required_argument, NULL, 9}, + {"extra-basebackup-args", required_argument, NULL, 10}, + {"text-types", no_argument, NULL, 11}, + {"bidirectional", no_argument, NULL, 12}, + {"stall-timeout", required_argument, NULL, 13}, + {"max-wait", required_argument, NULL, 14}, + {"cleanup", no_argument, NULL, 15}, + {"force", no_argument, NULL, 16}, + {"postgresql-auto-conf", required_argument, NULL, 17}, + {NULL, 0, NULL, 0} + }; + + /* check for --help */ + if (argc > 1) + { + for (i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-?") == 0) + { + usage(); + exit(0); + } + } + } + + /* Option parsing and validation */ + while ((c = getopt_long(argc, argv, "D:n:sv", long_options, &optindex)) != -1) + { + switch (c) + { + case 'D': + data_dir = expand_tilde(pg_strdup(optarg)); + break; + case 'n': + ctx->subscriber_name = pg_strdup(optarg); + break; + case 1: + ctx->base_prov_connstr = pg_strdup(optarg); + break; + case 2: + ctx->base_sub_connstr = pg_strdup(optarg); + break; + case 3: + ctx->replication_sets = validate_replication_set_input(pg_strdup(optarg)); + break; + case 4: + ctx->postgresql_conf = validated_existing_path(optarg, "postgresql.conf"); + break; + case 5: + ctx->pg_hba_conf = validated_existing_path(optarg, "pg_hba.conf"); + break; + case 6: + ctx->recovery_conf = validated_existing_path(optarg, "recovery configuration"); + break; + case 'v': + verbosity++; + break; + case 's': + ctx->stop = true; + break; + case 7: + ctx->drop_slot_if_exists = true; + break; + case 8: + ctx->apply_delay = parse_checked_int(optarg, "apply-delay"); + break; + case 9: + ctx->databases = pg_strdup(optarg); + break; + case 10: + ctx->extra_basebackup_args = pg_strdup(optarg); + validate_extra_basebackup_args(ctx->extra_basebackup_args); + break; + case 11: + ctx->force_text_transfer = true; + break; + case 12: + ctx->bidir.enabled = true; + break; + case 13: + ctx->bidir.stall_timeout = parse_checked_int(optarg, "stall-timeout"); + if (ctx->bidir.stall_timeout <= 0) + die(_("--stall-timeout must be a positive integer")); + break; + case 14: + ctx->bidir.max_wait = parse_checked_int(optarg, "max-wait"); + if (ctx->bidir.max_wait < 0) + die(_("--max-wait must be a non-negative integer")); + break; + case 15: + ctx->bidir.cleanup_mode = true; + break; + case 16: + ctx->bidir.force_cleanup = true; + break; + case 17: + ctx->postgresql_auto_conf = validated_existing_path(optarg, "postgresql.auto.conf"); + break; + default: + fprintf(stderr, _("Unknown option\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* + * Sanity checks + */ + + if (data_dir == NULL) + { + fprintf(stderr, _("No data directory specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + else if (ctx->subscriber_name == NULL && !ctx->bidir.cleanup_mode) + { + fprintf(stderr, _("No subscriber name specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + if (ctx->bidir.cleanup_mode && !ctx->bidir.enabled) + die(_("--cleanup requires --bidirectional.\n")); + + if (ctx->bidir.force_cleanup && !ctx->bidir.cleanup_mode) + die(_("--force requires --cleanup.\n")); + + if (!ctx->bidir.cleanup_mode && (!ctx->base_prov_connstr || !strlen(ctx->base_prov_connstr))) + die(_("Provider connection string must be specified.\n")); + if (!ctx->bidir.cleanup_mode && + (!ctx->base_sub_connstr || !strlen(ctx->base_sub_connstr))) + die(_("Subscriber connection string must be specified: --subscriber-dsn " + "is used both for the tool's own connection to the newly " + "created node and, with --bidirectional, as the externally-" + "reachable address registered via spock.node_create() for " + "peers to connect back to it.\n")); + + if (ctx->apply_delay < 0) + die(_("Apply delay cannot be negative.\n")); + + if (ctx->apply_delay > MAX_APPLY_DELAY) + die(_("Apply delay cannot be more than %d.\n"), MAX_APPLY_DELAY); + + if (ctx->bidir.enabled) + { + /* + * n3 is joining an existing mesh, so its subscriptions must select + * exactly what the mesh already replicates; replication_sets is + * derived from the source's own subscriptions below instead. + */ + if (ctx->replication_sets != NULL) + die(_("--replication-sets cannot be combined with --bidirectional; " + "the joining node's replication sets are detected " + "automatically from the cluster it is joining.\n")); + } + else if (!ctx->replication_sets || !strlen(ctx->replication_sets)) + ctx->replication_sets = "default,default_insert_only,ddl_sql"; + + /* Build the manifest path from --pgdata */ + if (ctx->bidir.enabled || ctx->bidir.cleanup_mode) + { + snprintf(ctx->bidir_manifest_path, MAXPGPATH, + "%s/spock_bidirectional_manifest.json", data_dir); + ctx->bidir.manifest_path = ctx->bidir_manifest_path; + + /* + * Sidecar path for the source slot orphan-protection record (see the + * write near source-slot creation below) -- lives next to, not + * inside, data_dir, since data_dir must still be empty when this is + * first written (pg_basebackup requires an empty target directory). + */ + snprintf(ctx->bidir_pending_path, MAXPGPATH, + "%s.spock_bidir_pending.json", data_dir); + if (ctx->bidir.stall_timeout == 0) + ctx->bidir.stall_timeout = 600; + } +} + +/* + * Phase 2: if --cleanup was requested, read the manifest (or, failing + * that, the pending-cleanup sidecar), remove partial join state, and + * exit -- this phase never returns when ctx->bidir.cleanup_mode is set. + * A no-op otherwise. + */ +static void +run_cleanup_mode_if_requested(SubscriberCreateContext *ctx) +{ + char *sub_name = NULL; + char *db = NULL; + char *src_dsn = NULL; + + if (!ctx->bidir.cleanup_mode) + return; + + if (read_manifest(ctx->bidir.manifest_path, &ctx->bidir, &sub_name, &db, &src_dsn)) + exit(cleanup_partial_state(&ctx->bidir, sub_name, db, src_dsn, + ctx->bidir.force_cleanup) ? 0 : 1); + + /* + * No full manifest -- basebackup may never have completed. Fall back to + * the pending-cleanup sidecar written right after source slot creation, + * so a slot orphaned by a failed/interrupted backup is still reachable by + * --cleanup. + */ + if (read_manifest(ctx->bidir_pending_path, &ctx->bidir, &sub_name, &db, &src_dsn)) + /* cleanup_partial_state() removes the sidecar itself on success. */ + exit(cleanup_partial_state(&ctx->bidir, sub_name, db, src_dsn, + ctx->bidir.force_cleanup) ? 0 : 1); + + /* + * Neither record exists -- there's no slot/subscription bookkeeping to + * act on, e.g. because the run died before the pending sidecar was even + * written. But an orphaned data_dir can still be sitting there from that + * attempt, and --force is an explicit instruction to remove it: don't + * leave it behind just because there was nothing to read. + */ + if (ctx->bidir.force_cleanup && data_dir != NULL && data_dir[0] && + file_exists(data_dir)) + { + fprintf(stderr, + _("No manifest found at %s or %s; no slot/subscription " + "state to clean up, but --force was given -- removing " + "data directory %s.\n"), + ctx->bidir.manifest_path, ctx->bidir_pending_path, data_dir); + exit(remove_data_dir_if_forced(true) ? 0 : 1); + } + + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), + ctx->bidir.manifest_path, ctx->bidir_pending_path); + exit(0); +} + +/* + * Phase 3: resolve --databases/--provider-dsn into ctx->database_list, + * enforcing --bidirectional's single-database restriction. + */ +static void +resolve_database_targets(SubscriberCreateContext *ctx) +{ + /* Init random numbers used for slot suffixes, etc */ + srand(time(NULL)); + + /* Parse database list or connection string. */ + if (ctx->databases != NULL) + { + ctx->database_list = get_database_list(ctx->databases, &ctx->n_databases); + } + else + { + char *dbname = get_connstr_dbname(ctx->base_prov_connstr); + + if (!dbname) + die(_("Either provider connection string must contain database " + "name or --databases option must be specified.\n")); + + ctx->n_databases = 1; + ctx->database_list = palloc(ctx->n_databases * sizeof(char *)); + ctx->database_list[0] = dbname; + } + + /* + * Single database only: all join state is per-database, and the + * physical-backup/recovery path operates on one data directory. Reject a + * multi-database request rather than silently joining only + * database_list[0]. Separate from check_single_spock_database() below, + * which checks the instance for spock on other databases. + */ + if (ctx->bidir.enabled && ctx->n_databases > 1) + die(_("--bidirectional supports a single database only; " + "%d were named via --databases/--provider-dsn.\n"), + ctx->n_databases); + + ctx->slot_names = palloc(ctx->n_databases * sizeof(char *)); +} + +/* + * Phase 4: validate every resolved database's connection strings before + * doing anything expensive. ctx->prov_connstr/ctx->sub_connstr here are + * just scratch -- the values used by later phases are recomputed there. + */ +static void +validate_connection_strings(SubscriberCreateContext *ctx) +{ + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + if (!ctx->prov_connstr || !strlen(ctx->prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + if (!ctx->bidir.enabled) + { + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + if (!ctx->sub_connstr || !strlen(ctx->sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } + } +} + +/* + * Phase 5: create the postgres startup log file, then, for each + * resolved database, connect to the provider and create a replication + * slot. --bidirectional additionally discovers peers, checks join + * preconditions, and persists a pending-cleanup sidecar before the base + * backup even starts -- then breaks after database_list[0], since + * --bidirectional is single-database only (enforced in + * resolve_database_targets()). Sets ctx->remote_info (the last/only + * database processed) and ctx->slot_names. + */ +static void +create_replication_slots(SubscriberCreateContext *ctx) +{ + int dbnum; + int logfd; + + /* + * Create log file where new postgres instance will log to while being + * initialized. + */ + logfd = open("spock_create_subscriber_postgres.log", O_CREAT | O_RDWR, + S_IRUSR | S_IWUSR); + if (logfd == -1) + { + die(_("Creating spock_create_subscriber_postgres.log failed: %s"), + strerror(errno)); + } + /* Safe to close() unchecked, we didn't write */ + (void) close(logfd); + + /* Let's start the real work... */ + print_msg(VERBOSITY_NORMAL, _("%s: starting ...\n"), progname); + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + if (!ctx->prov_connstr || !strlen(ctx->prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + /* Read the remote server indetification. */ + print_msg(VERBOSITY_NORMAL, + _("Getting information for database %s ...\n"), db); + provider_conn = connectdb(ctx->prov_connstr); + ctx->remote_info = get_remote_info(provider_conn); + + /* + * --bidirectional: discover peers, verify preconditions, then + * continue into the physical-backup pipeline below using the + * "sub__" slot naming convention. Manifest write + * is deferred until after the basebackup; see the comment there. + */ + if (ctx->bidir.enabled) + { + char *source_sub_name; + + /* + * Inherit the replication sets already in use by the cluster + * being joined, rather than accept a separately specified list -- + * see the die() in parse_options() that rejects + * --replication-sets together with --bidirectional. + */ + ctx->replication_sets = get_source_mesh_replication_sets(provider_conn); + print_msg(VERBOSITY_VERBOSE, + _("Replication sets inherited from the existing " + "cluster: %s\n"), ctx->replication_sets); + + ctx->bidir.num_peers = discover_peer_nodes(provider_conn, + ctx->remote_info->node_name, + ctx->subscriber_name, db, + &ctx->bidir.peers); + { + int pi; + + for (pi = 0; pi < ctx->bidir.num_peers; pi++) + print_msg(VERBOSITY_DEBUG, + _("Discovered peer \"%s\" (dsn \"%s\", slot \"%s\")\n"), + ctx->bidir.peers[pi].node_name, ctx->bidir.peers[pi].dsn, + ctx->bidir.peers[pi].slot_name); + } + check_preconditions(provider_conn, ctx->remote_info->node_name, + ctx->bidir.peers, ctx->bidir.num_peers); + check_single_spock_database(provider_conn, ctx->base_prov_connstr, db); + check_no_native_subscriptions(provider_conn); + ctx->use_existing_data_dir = check_data_dir(data_dir, ctx->remote_info); + if (ctx->use_existing_data_dir) + check_reused_data_dir_is_safe(data_dir, ctx->remote_info); + + source_sub_name = sub_name_for(ctx->subscriber_name, ctx->remote_info->node_name); + + print_msg(VERBOSITY_NORMAL, + _("Creating source replication slot in database %s ...\n"), db); + print_msg(VERBOSITY_DEBUG, + _("Creating replication slot on source \"%s\" for future " + "subscription \"%s\"\n"), ctx->remote_info->node_name, source_sub_name); + ctx->bidir.source_slot_name = initialize_replication_slot(provider_conn, + ctx->remote_info->dbname, + ctx->remote_info->node_name, + source_sub_name, + ctx->drop_slot_if_exists); + print_msg(VERBOSITY_DEBUG, _("Source replication slot created: \"%s\"\n"), + ctx->bidir.source_slot_name); + pg_free(source_sub_name); + + /* + * Persist a pending-cleanup record now, before the base backup + * even starts: the source slot above already exists on the remote + * node, and a failed/interrupted backup would otherwise orphan it + * with nothing for --cleanup to find (the real manifest can't be + * written yet -- data_dir must stay empty for pg_basebackup). + * Superseded and removed once the real manifest is written below. + */ + ctx->bidir.manifest_path = ctx->bidir_pending_path; + write_manifest(&ctx->bidir, ctx->subscriber_name, db, ctx->base_prov_connstr); + ctx->bidir.manifest_path = ctx->bidir_manifest_path; + + PQfinish(provider_conn); + provider_conn = NULL; + break; /* single-database only, enforced above */ + } + + /* only need to do this piece once */ + + if (dbnum == 0) + { + ctx->use_existing_data_dir = check_data_dir(data_dir, ctx->remote_info); + + if (ctx->use_existing_data_dir) + check_reused_data_dir_is_safe(data_dir, ctx->remote_info); + } + + /* + * Create replication slots on remote node. + */ + print_msg(VERBOSITY_NORMAL, + _("Creating replication slot in database %s ...\n"), db); + ctx->slot_names[dbnum] = initialize_replication_slot(provider_conn, + ctx->remote_info->dbname, + ctx->remote_info->node_name, + ctx->subscriber_name, + ctx->drop_slot_if_exists); + PQfinish(provider_conn); + provider_conn = NULL; + } +} + +/* + * Phase 6: take (or reuse) the physical base backup into data_dir, then + * write the real manifest -- deferred until here because pg_basebackup + * requires an empty target directory, and a manifest file in data_dir + * earlier would make it look non-empty. The pending-cleanup sidecar + * written in create_replication_slots() covers the gap between then and + * now; it's superseded and removed here. + */ +static void +run_basebackup_and_write_manifest(SubscriberCreateContext *ctx) +{ + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, ctx->database_list[0]); + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, ctx->database_list[0]); + + if (!ctx->use_existing_data_dir) + print_msg(VERBOSITY_DEBUG, + _("Taking a physical base backup from \"%s\" into \"%s\"\n"), + ctx->prov_connstr, data_dir); + else + print_msg(VERBOSITY_DEBUG, + _("Reusing existing data directory \"%s\" (already a basebackup " + "of this source)\n"), data_dir); + initialize_data_dir(data_dir, + ctx->use_existing_data_dir ? NULL : ctx->prov_connstr, + ctx->postgresql_conf, ctx->postgresql_auto_conf, ctx->pg_hba_conf, + ctx->extra_basebackup_args); + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + + if (ctx->bidir.enabled) + { + write_manifest(&ctx->bidir, ctx->subscriber_name, ctx->database_list[0], ctx->base_prov_connstr); + if (unlink(ctx->bidir_pending_path) != 0 && errno != ENOENT) + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove superseded pending sidecar " + "%s: %s\n"), ctx->bidir_pending_path, strerror(errno)); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "source slot created, manifest written to %s.\n"), + ctx->bidir.num_peers, ctx->bidir.manifest_path); + } +} + +/* + * Phase 7: create a restore point on the provider, bring the subscriber + * up to it via physical recovery, and wait for it to start accepting + * connections -- proof it has caught up. Sets ctx->remote_lsn. + */ +static void +catchup_to_restore_point(SubscriberCreateContext *ctx) +{ + PQExpBuffer recoveryconfcontents = createPQExpBuffer(); + char *restore_point_name; + int pg_ctl_ret; + + restore_point_name = generate_restore_point_name(); + + print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), + restore_point_name); + provider_conn = connectdb(ctx->prov_connstr); + ctx->remote_lsn = create_restore_point(provider_conn, restore_point_name); + PQfinish(provider_conn); + provider_conn = NULL; + + /* + * Get subscriber db to consistent state (for lsn after slot creation). + */ + print_msg(VERBOSITY_NORMAL, + _("Bringing subscriber node to the restore point ...\n")); + if (ctx->recovery_conf) + { + CopyConfFile(ctx->recovery_conf, "postgresql.auto.conf", true); + } + else + { + appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", + escape_single_quotes_ascii(ctx->prov_connstr)); + } + appendPQExpBuffer(recoveryconfcontents, "recovery_target_name = '%s'\n", restore_point_name); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_inclusive = true\n"); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_action = promote\n"); + WriteRecoveryConf(recoveryconfcontents); + + free(restore_point_name); + restore_point_name = NULL; + + /* + * Start subscriber node with spock disabled, and wait until it starts + * accepting connections which means it has caught up to the restore + * point. + * + * TODO: for --bidirectional this node should be network-quarantined + * (private socket/listen address, or a restrictive pg_hba.conf) from this + * first startup through the end of the join -- spock.readonly = 'local' + * (set later) blocks writes but not reads or peer probes. Not + * implemented: --subscriber-dsn must be directly reachable, and the + * tool's own connections use that same DSN throughout, so restricting + * listen_addresses here would also lock the tool itself out. + */ + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + + wait_primary_connection(ctx->sub_connstr, + ctx->bidir.enabled ? ctx->bidir.stall_timeout : 0, + ctx->bidir.enabled ? ctx->bidir.max_wait : 0); +} + +/* + * Phase 8: strip the spock configuration pg_basebackup copied over from + * the provider -- for --bidirectional, this also gives n3 its own + * system identifier, verifies --subscriber-dsn actually reaches it, and + * captures repset/table/sequence state before the catalog strip (sets + * ctx->bidir.node_sysid, ctx->capture); for the plain path, just clears + * each database's spock configuration. Ends by stopping postgres so it + * can be restarted with spock loaded. + */ +static void +strip_subscriber_catalog(SubscriberCreateContext *ctx) +{ + int pg_ctl_ret; + + /* + * Clean any per-node data that were copied by pg_basebackup. + */ + print_msg(VERBOSITY_VERBOSE, + _("Removing old spock configuration ...\n")); + + if (ctx->bidir.enabled) + { + Oid source_nodeid; + char *expected_sysid; + + /* + * Give n3 its own permanent identity now, right after promotion and + * before any catalog mutation: a physical backup preserves the + * source's system identifier, which risks stray WAL from one cluster + * being mistaken for the other's, and until reset makes + * system_identifier useless for proving a connection actually reaches + * n3 rather than the source. + */ + print_msg(VERBOSITY_NORMAL, + _("Assigning a new system identifier to the subscriber node...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop before resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + + { + sigset_t block_set, + old_set; + + /* + * Neither step below is safe to interrupt -- both write + * pg_control/WAL directly, and signal_handler() -> die() is not + * async-signal-safe. A signal landing mid-write could corrupt + * pg_control with no repair short of --cleanup --force. Block + * both signals across this pair of calls; any that arrives is + * deferred until right after. + */ + sigemptyset(&block_set); + sigaddset(&block_set, SIGINT); + sigaddset(&block_set, SIGTERM); + sigprocmask(SIG_BLOCK, &block_set, &old_set); + + expected_sysid = reset_subscriber_sysid(data_dir); + run_pg_resetwal(data_dir); + + sigprocmask(SIG_SETMASK, &old_set, NULL); + } + + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup after resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_connection(ctx->sub_connstr); + + subscriber_conn = connectdb(ctx->sub_connstr); + + /* + * --subscriber-dsn is expected to point directly at this node; verify + * that cheaply before running anything destructive, rather than + * trusting it silently. Now that n3 has just been given its own + * system identifier above, a straightforward comparison is a valid + * proof the connection reaches n3 and not the source or any other + * server -- unlike before the reset, nothing else could share it. + */ + { + PGresult *sysid_res = debug_exec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + bool mismatch; + + if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) + { + PQclear(sysid_res); + die(_("could not verify --subscriber-dsn connects to this node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + mismatch = strcmp(PQgetvalue(sysid_res, 0, 0), expected_sysid) != 0; + PQclear(sysid_res); + if (mismatch) + die(_("--subscriber-dsn does not connect to the node at \"%s\": " + "system identifier mismatch. This can happen if the DSN " + "routes to the source node or another server; refusing " + "to run catalog operations against it.\n"), data_dir); + } + + /* + * Persist n3's own system identifier so --cleanup can re-verify + * node_dsn still reaches this same node later, rather than trusting a + * possibly stale manifest to still point at the right server. + */ + ctx->bidir.node_sysid = expected_sysid; + + /* Capture repset/table/sequence state before the catalog strip. */ + source_nodeid = get_local_node_id(subscriber_conn); + print_msg(VERBOSITY_DEBUG, + _("Capturing replication-set/table/sequence membership for local " + "node id %u before dropping the spock extension\n"), source_nodeid); + capture_catalog_state(subscriber_conn, source_nodeid, &ctx->capture); + print_msg(VERBOSITY_DEBUG, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s)\n"), + ctx->capture.num_repsets, ctx->capture.num_tables, ctx->capture.num_sequences); + + /* Drop all origins, then guarded DROP EXTENSION. */ + print_msg(VERBOSITY_DEBUG, + _("Dropping replication origins and the spock extension (checking " + "pg_depend first for non-spock objects CASCADE would collaterally " + "drop)\n")); + remove_unwanted_data_bidir(subscriber_conn, &ctx->capture); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + else + { + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + + if (!ctx->sub_connstr || !strlen(ctx->sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(ctx->sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } + + /* + * Stop Postgres so we can start it again with spock + * (shared_preload_libraries) loaded. + */ + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + wait_postmaster_shutdown(); +} + +/* + * Phase 9: restart postgres with spock loaded, then bring the node + * live -- for --bidirectional, the full go-live pipeline (local node + * creation, readonly, replication-set restore, catchup subscription, + * coverage barrier, forwarding clear, direct/reverse subscriptions, + * dataflow verification, readonly lift); for the plain path, the + * ordinary extension/origin/subscription creation per database. + */ +static void +restart_with_spock_and_activate(SubscriberCreateContext *ctx) +{ + int pg_ctl_ret; + + /* + * Start the node again, now with spock active so that we can start the + * logical replication. This is final start, so don't log to to special + * log file anymore. + */ + print_msg(VERBOSITY_NORMAL, + _("Initializing spock on the subscriber node:\n")); + + pg_ctl_ret = run_pg_ctl("start"); + if (pg_ctl_ret != 0) + die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); + wait_postmaster_connection(ctx->bidir.enabled ? ctx->sub_connstr : ctx->base_sub_connstr); + + if (ctx->bidir.enabled) + { + char *db = ctx->database_list[0]; + + subscriber_conn = connectdb(ctx->sub_connstr); + + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the local node, then immediately go read-only -- no window + * where n3 is reachable/writable before that lands. No origin + * creation here; the catchup subscription creates it later. + * + * dsn is --subscriber-dsn (sub_connstr) -- the externally-reachable + * address other nodes use to connect back, not a separate --node-dsn + * option. + */ + print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), + ctx->subscriber_name); + print_msg(VERBOSITY_DEBUG, _("Registering node \"%s\" with dsn \"%s\"\n"), + ctx->subscriber_name, ctx->sub_connstr); + { + PQExpBuffer nodequery = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(nodequery, + "SELECT spock.node_create(node_name := %s, dsn := %s)", + PQescapeLiteral(subscriber_conn, ctx->subscriber_name, + strlen(ctx->subscriber_name)), + PQescapeLiteral(subscriber_conn, ctx->sub_connstr, + strlen(ctx->sub_connstr))); + res = debug_exec(subscriber_conn, nodequery->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create local node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + destroyPQExpBuffer(nodequery); + } + + print_msg(VERBOSITY_NORMAL, _("Setting spock.readonly = 'local'...\n")); + set_spock_readonly(subscriber_conn, "local"); + + /* Restore what was captured before the catalog strip. */ + print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + print_msg(VERBOSITY_DEBUG, + _("Restoring %d replication set(s), %d table membership(s), " + "%d sequence(s) onto node \"%s\"\n"), + ctx->capture.num_repsets, ctx->capture.num_tables, ctx->capture.num_sequences, + ctx->subscriber_name); + restore_replication_sets(subscriber_conn, &ctx->capture); + + ctx->bidir.source_restore_lsn = pg_strdup(ctx->remote_lsn); + ctx->bidir.node_dsn = ctx->sub_connstr; + write_manifest(&ctx->bidir, ctx->subscriber_name, db, ctx->base_prov_connstr); + + { + char *source_sub_name; + char *target_lsn; + + source_sub_name = sub_name_for(ctx->subscriber_name, ctx->remote_info->node_name); + + print_msg(VERBOSITY_NORMAL, _("Creating catchup subscription to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Creating subscription \"%s\" to source \"%s\" using slot " + "\"%s\", forward_origins={all}, enabled=false\n"), + source_sub_name, ctx->prov_connstr, ctx->bidir.source_slot_name); + create_catchup_subscription(subscriber_conn, source_sub_name, ctx->prov_connstr, + ctx->replication_sets, ctx->bidir.source_slot_name, + ctx->bidir.source_restore_lsn); + print_msg(VERBOSITY_DEBUG, + _("Subscription \"%s\" created, origin advanced to %s, and " + "enabled\n"), source_sub_name, ctx->bidir.source_restore_lsn); + + print_msg(VERBOSITY_NORMAL, _("Creating disabled peer subscriptions...\n")); + create_disabled_peer_subscriptions(subscriber_conn, ctx->bidir.peers, + ctx->bidir.num_peers, ctx->replication_sets); + + print_msg(VERBOSITY_NORMAL, _("Getting catchup target from the source...\n")); + target_lsn = get_catchup_target_lsn(ctx->prov_connstr); + print_msg(VERBOSITY_DEBUG, _("Catchup target LSN: %s\n"), target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Waiting for catchup to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Waiting for subscription \"%s\" (origin \"%s\") to reach " + "LSN %s\n"), source_sub_name, ctx->bidir.source_slot_name, target_lsn); + wait_for_catchup(subscriber_conn, source_sub_name, ctx->bidir.source_slot_name, + target_lsn, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + pg_free(target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Establishing peer coverage barrier...\n")); + establish_peer_coverage_barrier(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, source_sub_name, + ctx->bidir.source_slot_name, ctx->subscriber_name, db, + ctx->base_prov_connstr, ctx->bidir.stall_timeout, + ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Clearing forwarding on the catchup subscription...\n")); + clear_forwarding(subscriber_conn, ctx->prov_connstr, source_sub_name, + ctx->bidir.source_slot_name, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Enabling direct peer subscriptions...\n")); + enable_peer_subs(subscriber_conn, ctx->bidir.peers, ctx->bidir.num_peers, + ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Creating reverse subscriptions...\n")); + create_reverse_subscriptions(&ctx->bidir, ctx->subscriber_name, ctx->sub_connstr, + ctx->replication_sets, ctx->prov_connstr, + ctx->remote_info->node_name, db, ctx->base_prov_connstr); + + print_msg(VERBOSITY_NORMAL, _("Waiting for reverse subscriptions to be ready...\n")); + wait_for_reverse_subs_ready(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, ctx->subscriber_name, + ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Verifying bidirectional replication...\n")); + verify_bidirectional_dataflow(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, source_sub_name, + ctx->subscriber_name, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + pg_free(source_sub_name); + + print_msg(VERBOSITY_NORMAL, _("Lifting read-only mode...\n")); + set_spock_readonly(subscriber_conn, "off"); + } + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + + print_msg(VERBOSITY_NORMAL, + _("Bidirectional join complete: node \"%s\" is a live, verified " + "bidirectional member of the cluster.\n"), + ctx->subscriber_name); + } + else + { + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + + subscriber_conn = connectdb(ctx->sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which + * we already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, ctx->slot_names[dbnum], ctx->remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + ctx->subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), ctx->replication_sets); + + spock_subscribe(subscriber_conn, ctx->subscriber_name, ctx->sub_connstr, + ctx->prov_connstr, ctx->replication_sets, ctx->apply_delay, + ctx->force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } +} + +int +main(int argc, char **argv) +{ + SubscriberCreateContext ctx = {0}; + + argv0 = argv[0]; + progname = get_progname(argv[0]); + pg_logging_init(argv[0]); + start_time = time(NULL); + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + parse_options(argc, argv, &ctx); + run_cleanup_mode_if_requested(&ctx); /* exits if --cleanup was given */ + + resolve_database_targets(&ctx); + validate_connection_strings(&ctx); + create_replication_slots(&ctx); + run_basebackup_and_write_manifest(&ctx); + catchup_to_restore_point(&ctx); + strip_subscriber_catalog(&ctx); + restart_with_spock_and_activate(&ctx); + + /* If user does not want the node to be running at the end, stop it. */ + if (ctx.stop) + { + int pg_ctl_ret; + + print_msg(VERBOSITY_NORMAL, _("Stopping the subscriber node ...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Stopping postgres after successful subscribtion failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, _("All done\n")); + + return 0; +} + + +/* + * Print help. + */ +static void +usage(void) +{ + printf(_("%s create new spock subscriber from basebackup of provider.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]...\n"), progname); + printf(_("\nGeneral options:\n")); + printf(_(" -D, --pgdata=DIRECTORY data directory to be used for new node,\n")); + printf(_(" can be either empty/non-existing directory,\n")); + printf(_(" or directory populated using\n")); + printf(_(" pg_basebackup -X stream command\n")); + printf(_(" --databases optional list of databases to replicate\n")); + printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber;\n")); + printf(_(" with --bidirectional, also the externally-\n")); + printf(_(" reachable address peers use to connect back\n")); + printf(_(" to this node once joined (required)\n")); + printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); + printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); + printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); + printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); + printf(_(" -s, --stop stop the server once the initialization is done\n")); + printf(_(" -v increase logging verbosity; repeatable --\n")); + printf(_(" -v -v also traces every query this tool\n")); + printf(_(" runs, with its result status\n")); + printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); + printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_(" --text-types transfer column values as text rather than binary\n")); + printf(_(" (use when provider and subscriber differ in type\n")); + printf(_(" representation or endianness)\n")); + printf(_("\nConfiguration files override:\n")); + printf(_(" --hba-conf path to the new pg_hba.conf\n")); + printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --postgresql-auto-conf settings to override in postgresql.auto.conf\n")); + printf(_(" --recovery-conf path to the template recovery configuration\n")); + printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); + printf(_(" --bidirectional enable bidirectional join plumbing\n")); + printf(_(" --stall-timeout=SECS once PostgreSQL accepts connections, seconds of no\n")); + printf(_(" replay progress before giving up (default 600); does\n")); + printf(_(" not bound PostgreSQL's own startup\n")); + printf(_(" --max-wait=SECS hard ceiling on post-connection catchup wait, seconds\n")); + printf(_(" (default: unbounded); does not bound PostgreSQL's own\n")); + printf(_(" startup\n")); + printf(_(" --cleanup idempotently remove partial join state and exit;\n")); + printf(_(" stops postgres if it is running in --pgdata\n")); + printf(_(" --force with --cleanup, also remove the data directory\n")); + printf(_("\nDuring the join, this node must be network-quarantined (private address /\n")); + printf(_("restrictive pg_hba.conf) by the operator -- via --hba-conf/--postgresql-conf --\n")); + printf(_("until the join completes; the tool does not manage this for you.\n")); +} + +/* + * Print error and exit. + */ +static void +die(const char *fmt,...) +{ + va_list argptr; + + va_start(argptr, fmt); + vfprintf(stderr, fmt, argptr); + va_end(argptr); + + if (subscriber_conn) + PQfinish(subscriber_conn); + if (provider_conn) + PQfinish(provider_conn); + + if (get_pgpid()) + { + if (!run_pg_ctl("stop -s")) + { + fprintf(stderr, _("WARNING: postgres seems to be running, but could not be stopped\n")); + } + } + + exit(1); +} + +/* + * Print message to stdout and flush + */ +static void +print_msg(VerbosityLevelEnum level, const char *fmt,...) +{ + if (verbosity >= level) + { + va_list argptr; + + va_start(argptr, fmt); + vfprintf(stdout, fmt, argptr); + va_end(argptr); + fflush(stdout); + } +} + +/* + * PQexec() wrapper that logs the query text at VERBOSITY_DEBUG (-v -v) + * before running it, and the resulting status/row count after -- a + * drop-in replacement so every query this tool issues is traceable + * without a separate print_msg() call at each site. Callers still do + * their own PQresultStatus()/die() handling on the result exactly as + * with a plain PQexec() call. + */ +static PGresult * +debug_exec(PGconn *conn, const char *query) +{ + PGresult *res; + + print_msg(VERBOSITY_DEBUG, _(" > %s\n"), query); + res = PQexec(conn, query); + if (verbosity >= VERBOSITY_DEBUG) + { + if (PQresultStatus(res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_DEBUG, _(" < %s (%d row(s))\n"), + PQresStatus(PQresultStatus(res)), PQntuples(res)); + else + print_msg(VERBOSITY_DEBUG, _(" < %s\n"), + PQresStatus(PQresultStatus(res))); + } + + return res; +} + + +/* + * Start pg_ctl with given argument(s) - used to start/stop postgres + * + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * signal this call will die and not return. + */ +static int +run_pg_ctl(const char *arg) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); + + appendPQExpBuffer(cmd, "\"%s\" %s -D \"%s\"", exec_path, arg, data_dir); + + /* Run pg_ctl in silent mode unless we run in debug mode. */ + if (verbosity < VERBOSITY_DEBUG) + appendPQExpBuffer(cmd, " -s"); + + print_msg(VERBOSITY_DEBUG, _("Running pg_ctl: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret)) + return WEXITSTATUS(ret); + else if (WIFSIGNALED(ret)) + die(_("pg_ctl exited with signal %d"), WTERMSIG(ret)); + else + die(_("pg_ctl exited for an unknown reason (system() returned %d)"), ret); + + return -1; +} + + +/* + * Reject --extra-basebackup-args values containing shell control + * characters. The args are appended to a system() command string, so + * semicolons, pipes, backticks, and similar metacharacters would allow + * arbitrary command injection. + */ +static void +validate_extra_basebackup_args(const char *args) +{ + const char *p; + + for (p = args; *p; p++) + { + if (*p == ';' || *p == '|' || *p == '&' || *p == '`' || + *p == '$' || *p == '(' || *p == ')' || + *p == '<' || *p == '>' || *p == '{' || *p == '}' || + *p == '\n' || *p == '\r') + die(_("--extra-basebackup-args contains unsafe shell characters\n")); + } +} + +/* + * Parse optarg as a base-10 integer, dying with a message naming + * opt_name if any part of it isn't numeric or the value doesn't fit in + * an int -- shared by every --