diff --git a/.gitignore b/.gitignore index b792ef5a..a798543d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ results regression_output tmp_check -spock_create_subscriber +/utils/spock_create_subscriber/spock_create_subscriber +/utils/spock_create_subscriber/.deps/ .vimrc *.o *.so diff --git a/Makefile b/Makefile index 1105a4fb..a4c63dbe 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ EXTENSION = spock PGFILEDESC = "spock - multi-master replication" MODULES = spock_output +SUBDIRS = utils/spock_create_subscriber # Lookup source directory vpath % src src/compat/$(PGVER) diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md new file mode 100644 index 00000000..64528071 --- /dev/null +++ b/docs/creating_subscriber_nodes.md @@ -0,0 +1,38 @@ +## Creating a Subscriber Node with pg_basebackup + +Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. + +### Synopsis: + + `spock_create_subscriber [OPTION]...` + +**Options** + +Specify the following options as needed. + +| Option | Description +|----------|------------- +| `-D`, `--pgdata=DIRECTORY` | The `data` directory to be used for new node. This can be either an empty/non-existing directory, or a directory populated using the `pg_basebackup -X stream` command. +| `--databases` | An optional list of databases to replicate. +| `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. +| `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. +| `--provider-dsn=CONNSTR` | A connection string to the provider. +| `--replication-sets=SETS` | A comma-separated list of replication set names. +| `--apply-delay=DELAY` | The apply delay in seconds (by default 0). +| `--drop-slot-if-exists` | Drop replication slot of conflicting name. +| `-s`, `--stop` | Stop the server once the initialization is done. +| `-v` | Increase logging verbosity. +| `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` +| `--text-types` | Transfer all column values as text rather than binary during initial sync. Use this when provider and subscriber differ in endianness or type representation. + +**Configuration files overrides** + +You can use the following options to override the location of the configuration files. + +| Option | Description +|----------|------------- +|`--hba-conf` | path to the new `pg_hba.conf` +| `--postgresql-conf` | path to the new `postgresql.conf` +| `--recovery-conf` | path to the template recovery configuration + +Unlike `spock.sub_create`'s other data sync options, this method of cloning ignores replication sets and copies all tables on all databases. However, it's often much faster, especially over high-bandwidth connections. diff --git a/src/spock_fe.c b/src/spock_fe.c index ebe62a82..0a6d66e0 100644 --- a/src/spock_fe.c +++ b/src/spock_fe.c @@ -226,10 +226,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -238,6 +240,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) diff --git a/tests/tap/schedule b/tests/tap/schedule index 7e8c457e..f71dfd37 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,6 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts +test: 048_bidir_join # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # @@ -65,3 +66,4 @@ test: 045_lsn_from_commit_ts # Regression tests test: 103_manager_worker_dboid_race test: 105_sub_disable_retransmit_after_disconnect + diff --git a/tests/tap/t/048_bidir_join.pl b/tests/tap/t/048_bidir_join.pl new file mode 100644 index 00000000..770a93de --- /dev/null +++ b/tests/tap/t/048_bidir_join.pl @@ -0,0 +1,759 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 048_bidir_join.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the bidirectional node-join procedure end to end: physical +# backup, recovery to a restore point, catalog strip (capture + origin drop +# + guarded DROP EXTENSION), replication-set/table/sequence restore, and +# catchup (disabled-first subscription to the source, disabled placeholder +# subscriptions to every peer, and a wait for n3 to reach a target LSN +# captured on the source) -- stopping short of enabling any direct peer +# subscription (a later step). +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node +# cluster from create_cluster/cross_wire) +# n3 a real third PostgreSQL instance built via +# `spock_create_subscriber --bidirectional`, physically backed +# up from n1. +# +# Test count breakdown: +# 1 binary found +# 5 create_cluster(2) +# 1 cross_wire n1<->n2 +# 1 custom replication set created on n1 +# 1 table with row_filter added to custom set on n1 +# 1 table with explicit column list added to custom set on n1 +# 1 sequence added to custom set on n1 +# 1 sequence advanced past its initial value on n1 (setval fidelity check) +# 1 partitioned table (parent + 2 children) added to custom set on n1 +# 1 sequence with apostrophe in name added to custom set on n1 +# 1 peer-forwarding test table created on n1 +# 1 peer-forwarding test table replicated to n2 +# 1 --bidirectional exits 0 +# 1 n3 postgres is running +# 1 spock extension installed cleanly on n3 (exactly one row) +# 1 n3 has exactly the catchup and peer origins, none leftover from the basebackup +# 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 +# 1 spock.readonly is lifted on n3 once the join is fully verified +# 1 custom replication set restored on n3 with correct flags +# 1 table membership restored with correct row_filter +# 1 table membership restored with correct explicit column list +# 1 sequence value restored exactly (last_value) +# 1 sequence is_called restored exactly +# 1 sequence pr3_test_seq is a member of pr3_test_repset on n3 +# 1 partitioned table parent + 2 children all present in repset on n3 +# 1 apostrophe-named sequence value restored on n3 +# 1 apostrophe-named sequence is_called restored on n3 +# 1 apostrophe-named sequence is a member of pr3_test_repset on n3 +# 1 manifest: source_slot_name populated +# 1 manifest: source_restore_lsn populated +# 1 manifest: node_dsn populated +# 1 source slot exists on n1 +# 1 catchup subscription sub_n3_n1 is replicating on n3 +# 1 forwarding cleared on sub_n3_n1 after cutover +# 1 direct peer subscription sub_n3_n2 is replicating on n3 after cutover +# 1 peer slot created on n2 during the coverage barrier +# 1 n2's post-cutover write reached n3 via the direct sub_n3_n2 path +# 1 n3's origin for peer n2 advanced via the direct subscription +# 1 reverse subscription sub_n2_n3 is replicating on n2 +# 1 reverse subscription sub_n1_n3 is replicating on n1 +# 1 n3's post-join write reached the source via sub_n1_n3 +# 1 n3's post-join write reached the peer via sub_n2_n3 +# 1 --cleanup --force exits 0 +# 1 source slot removed from n1 after cleanup +# 1 n3 data directory removed after cleanup --force +# 1 manifest removed after cleanup +# 1 --bidirectional rejects a multi-database request +# 1 --max-wait rejects a value that overflows int +# 1 --stall-timeout rejects a non-numeric value +# 1 --apply-delay rejects a value that overflows int +# 1 --bidirectional aborts when another database on the source has spock configured +# 1 --bidirectional rejects a broken full-mesh topology (disabled subscription) +# 1 --bidirectional rejects mismatched replication-set flags between source and peer +# 1 a broken --extra-basebackup-args makes the base backup fail +# 1 pending-cleanup sidecar written before the failed backup +# 1 pending-cleanup sidecar is mode 0600 +# 1 source slot still exists on n1 after the failed backup (orphaned) +# 1 --cleanup --force recovers via the pending sidecar +# 1 source slot removed from n1 via sidecar-based cleanup +# 1 pending-cleanup sidecar removed after cleanup +# 1 a broken backup orphans a slot for the retry-cleanup test +# 1 pending sidecar written for the retry-cleanup test +# 1 --cleanup exits non-zero when the source is unreachable +# 1 pending sidecar retained after an incomplete cleanup +# 1 n1 postgres is running again +# 1 --cleanup --force succeeds once the source is reachable again +# 1 pending sidecar removed once cleanup actually completed +# 1 destroy_cluster +# --- +# 72 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 72; +use File::Path qw(remove_tree); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +my $n1_sysid = scalar_query(1, "SELECT system_identifier FROM pg_control_system()"); + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# Seed n1 with a custom replication set, a table with a row_filter, and a +# sequence, to exercise the catalog capture/restore with non-default state +# rather than just the three built-in sets. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_create('pr3_test_repset', true, true, true, false)"; +pass('custom replication set created on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_tbl (id serial primary key, region text, value integer)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_tbl', synchronize_data := false, " . + "row_filter := 'region = ''east''')"; +pass('table with row_filter added to custom set on n1'); + +# Table with an explicit, non-default column list, to exercise the +# columns := restore path (captured/restored as a bare array- +# literal string relying on implicit text[] coercion) -- previously +# untested, so a round-trip regression here could pass silently. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_cols (id serial primary key, region text, " . + "value integer, secret text)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_cols', synchronize_data := false, " . + "columns := ARRAY['id', 'region', 'value'])"; +pass('table with explicit column list added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE pr3_test_seq"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_seq('pr3_test_repset', 'pr3_test_seq')"; +pass('sequence added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT setval('pr3_test_seq', 42, true)"; +my $seq_before = scalar_query(1, "SELECT last_value FROM pr3_test_seq"); +is($seq_before, '42', 'sequence advanced past its initial value on n1'); + +# Partitioned table: parent + 2 children get separate captured membership +# rows (that's how include_partitions => true populated them here); restore +# must not try to re-add children a second time via the parent's own +# include_partitions => true call. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part (id int, region text, PRIMARY KEY (id, region)) PARTITION BY LIST (region)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_east PARTITION OF pr3_test_part FOR VALUES IN ('east')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_west PARTITION OF pr3_test_part FOR VALUES IN ('west')"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_part', synchronize_data := false, " . + "include_partitions := true)"; +pass('partitioned table (parent + 2 children) added to custom set on n1'); + +# Sequence with an apostrophe in its name, to exercise setval() quoting. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(CREATE SEQUENCE "weird's_seq"); +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT spock.repset_add_seq('pr3_test_repset', '"weird''s_seq"')); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT setval('"weird''s_seq"', 7, true)); +pass('sequence with apostrophe in name added to custom set on n1'); + +# Table used later to verify n3's origin for peer n2 advances via forwarding. +# Created on n1 only and left to arrive on n2 via DDL replication (creating +# it directly on both sides races the already-established cross-wire DDL +# replay); spock.include_ddl_repset=on adds it to 'default' on each node +# once it lands there. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr4_peer_tbl (id serial primary key, val text)"; +pass('peer-forwarding test table created on n1'); + +my $tbl_on_n2 = '0'; +for (1 .. 15) { + $tbl_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'pr4_peer_tbl'"); + last if $tbl_on_n2 eq '1'; + sleep(1); +} +is($tbl_on_n2, '1', 'peer-forwarding test table replicated to n2'); + +# check_preconditions() requires all of n1's outbound replication to have +# caught up (no unreplicated DDL/data still in flight to n2); wait for the +# setup above to drain. +for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + last if defined $lag && $lag eq '0'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional continues through physical backup / catalog strip / +# repset restore, stopping before the catchup subscription. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3'; +my $manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; + +# n3's postgresql.conf is copied verbatim from n1 by the basebackup, port and +# all -- since all nodes run on the same host in this test, n3 must be given +# an override with its own port (a real cross-host join wouldn't need this). +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + ], + '--bidirectional exits 0' +); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); + +my $ext_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_extension WHERE extname = 'spock'"`; +$ext_count =~ s/\s+//g; +is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); + +# By this point the catchup subscription and the one disabled peer +# subscription (n2) have each created their own origin -- exactly 2, not +# more. Anything beyond that would mean an origin survived from the +# basebackup instead of being dropped by the catalog strip. +my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; +$origin_count =~ s/\s+//g; +is($origin_count, '2', + 'n3 has exactly the catchup and peer origins, none leftover from the basebackup'); + +my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; +$n3_sysid =~ s/\s+//g; +isnt($n3_sysid, $n1_sysid, + 'n3 was given its own system identifier (pg_resetwal), distinct from n1'); + +my $readonly = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SHOW spock.readonly"`; +$readonly =~ s/\s+//g; +is($readonly, 'off', + "spock.readonly is lifted on n3 once the join is fully verified"); + +my $repset_flags = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT replicate_insert, replicate_update, replicate_delete, replicate_truncate FROM spock.replication_set WHERE set_name = 'pr3_test_repset'"`; +$repset_flags =~ s/\s+//g; +is($repset_flags, 't|t|t|f', 'custom replication set restored on n3 with correct flags'); + +my $row_filter = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT pg_get_expr(rts.set_row_filter, rts.set_reloid) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset'"`; +$row_filter =~ s/^\s+|\s+$//g; +like($row_filter, qr/region\s*=\s*'east'/, 'table membership restored with correct row_filter'); + +my $columns = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT rts.set_att_list FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text = 'pr3_test_cols'"`; +$columns =~ s/^\s+|\s+$//g; +is($columns, '{id,region,value}', + 'table membership restored with correct explicit column list'); + +my $seq_last_value = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT last_value FROM pr3_test_seq"`; +$seq_last_value =~ s/\s+//g; +is($seq_last_value, '42', 'sequence value restored exactly (last_value)'); + +my $seq_is_called = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT is_called FROM pr3_test_seq"`; +$seq_is_called =~ s/\s+//g; +is($seq_is_called, 't', 'sequence is_called restored exactly'); + +# pr3_test_seq must be an actual member of pr3_test_repset on n3, not just +# have its value restored (a regression here is the sequence-membership bug: +# setval() alone leaves the sequence unpublished). +my $seq_member = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = 'pr3_test_seq'"`; +$seq_member =~ s/\s+//g; +is($seq_member, '1', 'sequence pr3_test_seq is a member of pr3_test_repset on n3'); + +# Partitioned table: parent + 2 children must all be present as distinct +# memberships (a regression here is include_partitions => true re-adding +# already-captured children and violating the (set_id, set_reloid) PK, +# which would have aborted the join above rather than just miscounting). +my $part_member_count = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text LIKE 'pr3_test_part%'"`; +$part_member_count =~ s/\s+//g; +is($part_member_count, '3', 'partitioned table parent + 2 children all present in repset on n3'); + +# Sequence with an apostrophe in its name: value/is_called restored and +# membership present, without a SQL syntax error breaking the whole run. +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out =~ s/^\s+|\s+$//g if defined $out; + return $out; +} + +my $weird_seq_value = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT last_value FROM "weird\'s_seq"'); +is($weird_seq_value, '7', "apostrophe-named sequence value restored on n3"); + +my $weird_seq_called = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT is_called FROM "weird\'s_seq"'); +is($weird_seq_called, 't', "apostrophe-named sequence is_called restored on n3"); + +my $weird_seq_member = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = '\"weird''s_seq\"'"); +is($weird_seq_member, '1', "apostrophe-named sequence is a member of pr3_test_repset on n3"); + +# ============================================================================= +# Manifest content checks +# ============================================================================= +my $manifest_content = ''; +if (-f $manifest) { + open my $fh, '<', $manifest or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +ok($manifest_content =~ /"source_slot_name":\s*"[^"]+"/, + 'manifest: source_slot_name populated'); +ok($manifest_content =~ /"source_restore_lsn":\s*"[0-9A-Fa-f]+\/[0-9A-Fa-f]+"/, + 'manifest: source_restore_lsn populated'); +ok($manifest_content =~ /"node_dsn":\s*"[^"]+"/, + 'manifest: node_dsn populated'); + +my $source_slot_exists = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($source_slot_exists >= 1, 'source slot exists on n1'); + +# ============================================================================= +# TEST: catchup subscription and, after cutover, the direct peer +# subscription are both replicating; a post-cutover write on n2 reaches n3 +# via the direct path, advancing n3's origin for n2. +# ============================================================================= +my $sub_status = ''; +for (1 .. 30) { + $sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n1')"); + last if $sub_status eq 'replicating'; + sleep(1); +} +is($sub_status, 'replicating', 'catchup subscription sub_n3_n1 is replicating on n3'); + +is(psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT forward_origins FROM spock.sub_show_status('sub_n3_n1')"), + '', "forwarding cleared on sub_n3_n1 after cutover"); + +my $peer_sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n2')"); +is($peer_sub_status, 'replicating', + 'direct peer subscription sub_n3_n2 is replicating on n3 after cutover'); + +# Origin name matches what create_disabled_peer_subscriptions() computed for +# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')) -- the same +# value is also the slot name create_peer_slot() created on n2 during the +# coverage barrier. +my $n2_origin_name = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); + +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name = '$n2_origin_name'"), + '1', 'peer slot created on n2 during the coverage barrier'); + +my $n2_origin_query = + "SELECT COALESCE(s.remote_lsn::text, '0/0') FROM pg_replication_origin o " . + "LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id " . + "WHERE o.roname = '$n2_origin_name'"; + +# Write on n2 after cutover; forwarding is off and the direct sub_n3_n2 is +# enabled, so this reaches n3 directly from n2, not via n1. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (val) VALUES ('from_n2_post_join')"; + +my $row_on_n3 = '0'; +for (1 .. 30) { + $row_on_n3 = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n2_post_join'"); + last if $row_on_n3 eq '1'; + sleep(1); +} +is($row_on_n3, '1', "n2's post-cutover write reached n3 via the direct sub_n3_n2 path"); + +my $n2_origin_lsn = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced via the direct subscription"); + +# ============================================================================= +# TEST: reverse subscriptions are replicating, and a write on n3 reaches +# both the source and the peer through them -- an external proof, +# independent of the utility's own internal verify_bidirectional_dataflow() +# check. +# ============================================================================= +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n2_n3')"), + 'replicating', 'reverse subscription sub_n2_n3 is replicating on n2'); + +is(psql_capture('-p', $node_ports->[0], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n1_n3')"), + 'replicating', 'reverse subscription sub_n1_n3 is replicating on n1'); + +# Explicit id: pr4_peer_tbl's serial sequence isn't part of the custom +# repset that gets its value round-tripped onto n3 (only pr3_test_seq and +# the apostrophe-named sequence are), so n3's own local copy of the +# sequence is still at its basebackup-time value and would collide with +# the id the n2-post-join row already claimed via replication. +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (id, val) VALUES (1000, 'from_n3_post_join')"; + +my $row_on_n1 = '0'; +for (1 .. 30) { + $row_on_n1 = scalar_query(1, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n1 eq '1'; + sleep(1); +} +is($row_on_n1, '1', "n3's post-join write reached the source via sub_n1_n3"); + +my $row_on_n2_from_n3 = '0'; +for (1 .. 30) { + $row_on_n2_from_n3 = scalar_query(2, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n2_from_n3 eq '1'; + sleep(1); +} +is($row_on_n2_from_n3, '1', "n3's post-join write reached the peer via sub_n2_n3"); + +# ============================================================================= +# TEST: --cleanup --force removes source slot, data directory, and manifest +# ============================================================================= +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $n3_datadir, + ], + '--cleanup --force exits 0' +); + +my $source_slot_after = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); +ok(!-f $manifest, 'manifest removed after cleanup'); + +# ============================================================================= +# TEST: --bidirectional hard-rejects a multi-database request outright, +# rather than silently joining only the first-named database -- all join +# state is per-database. +# ============================================================================= +my $multidb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_multidb'; +remove_tree($multidb_datadir) if -d $multidb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $multidb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--databases', "$dbname,postgres"), + '--bidirectional rejects a multi-database request'); +remove_tree($multidb_datadir) if -d $multidb_datadir; + +# ============================================================================= +# TEST: option parsing rejects malformed/out-of-range integer arguments +# instead of silently truncating them -- e.g. a value like 4294967296 +# wraps to 0 when cast to int after strtol(), which would otherwise turn +# --max-wait=4294967296 into an accepted, effectively-unbounded wait. +# ============================================================================= +my $intcheck_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_intcheck'; +remove_tree($intcheck_datadir) if -d $intcheck_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--max-wait', '4294967296'), + '--max-wait rejects a value that overflows int'); + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--stall-timeout', 'abc'), + '--stall-timeout rejects a non-numeric value'); + +ok(!system_maybe($SCS_BIN, + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--apply-delay', '99999999999999999999'), + '--apply-delay rejects a value that overflows int'); +remove_tree($intcheck_datadir) if -d $intcheck_datadir; + +# ============================================================================= +# TEST: --bidirectional aborts if the source instance has spock configured +# on another database too, even though that database was never named via +# --databases (check_single_spock_database() must fail closed). +# ============================================================================= +system_or_bail "$pg_bin/createdb", '-p', $node_ports->[0], 'pr3_other_db'; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "CREATE EXTENSION spock"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_create('pr3_other_node', 'dbname=pr3_other_db')"; + +my $otherdb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_otherdb'; +remove_tree($otherdb_datadir) if -d $otherdb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $otherdb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional aborts when another database on the source has spock configured'); +remove_tree($otherdb_datadir) if -d $otherdb_datadir; + +system_maybe "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_drop('pr3_other_node')"; +system_maybe "$pg_bin/dropdb", '-p', $node_ports->[0], 'pr3_other_db'; + +# ============================================================================= +# TEST: --bidirectional rejects a broken full-mesh topology -- a disabled +# subscription is not a valid mesh edge, even though it still exists. A +# plain subscription COUNT would not catch this. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_disable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 'f'; + sleep(1); +} + +my $mesh_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_mesh'; +remove_tree($mesh_datadir) if -d $mesh_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $mesh_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects a broken full-mesh topology (disabled subscription)'); +remove_tree($mesh_datadir) if -d $mesh_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_enable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 't'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional rejects mismatched replication-set definitions for a +# selected (subscription-referenced) set between source and peer -- a +# repset the forwarding path and a future direct-peer path disagree on can +# permanently drop changes on cutover. DDL replication is disabled for the +# ALTER itself so the mismatch is real and local to n2. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := false)"; + +my $repset_mismatch_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_repset_mismatch'; +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $repset_mismatch_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects mismatched replication-set flags between source and peer'); +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := true)"; + +# ============================================================================= +# TEST: a failed base backup leaves the source slot recoverable via +# --cleanup, even though the real manifest was never written -- a +# pending-cleanup sidecar is persisted right after source slot creation, +# before the backup even starts, since data_dir must stay empty until +# pg_basebackup runs. +# ============================================================================= +my $failed_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_failed'; +my $pending_sidecar = "${failed_datadir}.spock_bidir_pending.json"; +remove_tree($failed_datadir) if -d $failed_datadir; +unlink($pending_sidecar) if -f $pending_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $failed_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_xyz'), + 'a broken --extra-basebackup-args makes the base backup fail'); + +ok(-f $pending_sidecar, 'pending-cleanup sidecar written before the failed backup'); + +my $sidecar_mode = (stat($pending_sidecar))[2] & 07777; +is(sprintf('%04o', $sidecar_mode), '0600', + 'pending-cleanup sidecar is mode 0600 (may carry a DSN password)'); + +my $slot_after_failed_backup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($slot_after_failed_backup >= 1, + 'source slot still exists on n1 after the failed backup (orphaned)'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $failed_datadir, + ], + '--cleanup --force recovers via the pending sidecar (no real manifest exists)' +); + +my $slot_after_sidecar_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($slot_after_sidecar_cleanup, '0', + 'source slot removed from n1 via sidecar-based cleanup'); + +ok(!-f $pending_sidecar, 'pending-cleanup sidecar removed after cleanup'); +remove_tree($failed_datadir) if -d $failed_datadir; + +# ============================================================================= +# TEST: an incomplete cleanup (source unreachable) exits non-zero and keeps +# the pending sidecar so it can be retried, instead of unconditionally +# deleting the only retry record. +# ============================================================================= +my $retry_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_retry'; +my $retry_sidecar = "${retry_datadir}.spock_bidir_pending.json"; +remove_tree($retry_datadir) if -d $retry_datadir; +unlink($retry_sidecar) if -f $retry_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $retry_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_retry'), + 'a broken backup orphans a slot for the retry-cleanup test'); +ok(-f $retry_sidecar, 'pending sidecar written for the retry-cleanup test'); + +my $n1_datadir = $config->{node_datadirs}->[0]; +system_or_bail "$pg_bin/pg_ctl", 'stop', '-D', $n1_datadir, '-m', 'fast'; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir), + '--cleanup exits non-zero when the source is unreachable'); +ok(-f $retry_sidecar, + 'pending sidecar retained after an incomplete cleanup (retryable)'); + +system_or_bail "$pg_bin/pg_ctl", 'start', '-D', $n1_datadir, + '-l', "$config->{log_dir}/n1_retry_restart.log"; +ok(wait_for_pg_ready($host, $node_ports->[0], $pg_bin, 30), + 'n1 postgres is running again'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir, + ], + '--cleanup --force succeeds once the source is reachable again' +); +ok(!-f $retry_sidecar, + 'pending sidecar removed once cleanup actually completed'); +remove_tree($retry_datadir) if -d $retry_datadir; + +# ============================================================================= +# CLEANUP +# ============================================================================= +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_tbl"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_part"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_cols"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP SEQUENCE IF EXISTS pr3_test_seq"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(DROP SEQUENCE IF EXISTS "weird's_seq"); +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_drop('pr3_test_repset')"; +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Cleanup'); diff --git a/utils/spock_create_subscriber/Makefile b/utils/spock_create_subscriber/Makefile new file mode 100644 index 00000000..54e39446 --- /dev/null +++ b/utils/spock_create_subscriber/Makefile @@ -0,0 +1,15 @@ +# Makefile for spock_create_subscriber utility +PG_CONFIG ?= pg_config +PROGRAM = spock_create_subscriber + +PG_CPPFLAGS = -I../../include -I$(shell $(PG_CONFIG) --includedir) +PG_LDFLAGS = -lpq -L$(shell $(PG_CONFIG) --libdir) + +# create symlink to spock_fe.c here +spock_fe.c: ../../src/spock_fe.c + ln -sf $< $@ +OBJS = spock_create_subscriber.o spock_fe.o + +# PGXS +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c new file mode 100644 index 00000000..e38642ff --- /dev/null +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -0,0 +1,6718 @@ +/* ------------------------------------------------------------------------- + * + * spock_create_subscriber.c + * Initialize a new spock subscriber from a physical base backup + * + * Copyright (c) 2022-2024, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + * ------------------------------------------------------------------------- + */ + +/* dirent.h on port/win32_msvc expects MAX_PATH to be defined */ +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Note the order is important for debian here. */ +#if !defined(pg_attribute_printf) + +/* GCC and XLC support format attributes */ +#if defined(__GNUC__) || defined(__IBMC__) +#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) +#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) +#else +#define pg_attribute_format_arg(a) +#define pg_attribute_printf(f,a) +#endif + +#endif + +#include "libpq-fe.h" +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +#include "getopt_long.h" + +#include "miscadmin.h" + +#include "access/timeline.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" +#include "common/controldata_utils.h" +#include "common/file_utils.h" +#include "common/jsonapi.h" +#include "common/logging.h" +#include "mb/pg_wchar.h" +#include "port.h" + +#include "spock_fe.h" + +#define MAX_APPLY_DELAY 86400 + +typedef struct RemoteInfo { + Oid nodeid; + char *node_name; + char *sysid; + char *dbname; + char *replication_sets; + TimeLineID timeline_id; /* current TLI, for detecting a data_dir + * left over from an already-promoted + * earlier attempt (see check_data_dir()) */ +} RemoteInfo; + +typedef struct PeerNodeInfo +{ + char *node_name; + char *dsn; + char *slot_name; /* from spock.spock_gen_slot_name() */ + char *sub_name; /* "sub__" */ + bool reverse_sub_created; + + /* + * slot_creation_lsn doubles as the "peer slot created" flag: a + * non-empty value means the slot exists, so no separate boolean is + * kept. It is persisted to the manifest for --cleanup; peer_marker_lsn + * below it is in-memory only -- it doesn't gate a --cleanup decision, + * and cutover has no cross-invocation resume in v1 (see + * check_reused_data_dir_is_safe()). + */ + char *slot_creation_lsn; /* peer's replication slot creation LSN */ + char *peer_marker_lsn; /* this peer's marker once seen on the source */ +} PeerNodeInfo; + +typedef struct BidirectionalState +{ + bool enabled; + int num_peers; + PeerNodeInfo *peers; + int stall_timeout; /* default 600s */ + int max_wait; /* default 0 = unbounded */ + char *source_slot_name; + char *source_restore_lsn; /* recovery target LSN; consumed by the + * disabled-first catchup sub_create. + * Written to the manifest as a + * diagnostic record of what was used, + * but never read back on a --cleanup + * reload -- nothing needs it restored + * into memory there */ + char *node_dsn; /* DSN registered via spock.node_create(); + * the address peers use to connect back to + * this node. Derived from --subscriber-dsn. */ + char *node_sysid; /* n3's system_identifier; lets --cleanup + * confirm node_dsn still reaches this node + * before dropping subscriptions there. */ + bool source_reverse_sub_created; /* sub__ created on + * the source -- must be + * persisted, unlike the peer + * progress bits, since it is + * the only way --cleanup knows + * to drop this specific + * subscription on a node that + * also has other, unrelated, + * legitimate subscriptions */ + bool cleanup_mode; + bool force_cleanup; /* --force: also remove the data directory + * on --cleanup, not just remote state */ + char *manifest_path; +} BidirectionalState; + +/* + * Replication-set / table-membership / sequence state captured from the + * source's catalog before DROP EXTENSION spock removes it. Utility-side + * memory only; never written to the manifest. + */ +typedef struct RepsetCapture +{ + char *set_name; + bool replicate_insert; + bool replicate_update; + bool replicate_delete; + bool replicate_truncate; +} RepsetCapture; + +typedef struct RepsetTableCapture +{ + char *set_name; + char *qualified_table; /* rts.set_reloid::regclass */ + char *columns; /* rts.set_att_list, NULL if all columns */ + char *row_filter; /* pg_get_expr(...), NULL if none */ +} RepsetTableCapture; + +typedef struct SequenceCapture +{ + char *set_name; + char *qualified_seq; + int64 last_value; + bool is_called; +} SequenceCapture; + +typedef struct CatalogCapture +{ + RepsetCapture *repsets; + int num_repsets; + RepsetTableCapture *tables; + int num_tables; + SequenceCapture *sequences; + int num_sequences; +} CatalogCapture; + +/* + * main()'s working state, threaded by pointer through each lifecycle + * phase function below (parse_options() through + * restart_with_spock_and_activate()) in the same order those phases run. + * Holds every option and every piece of state one phase sets that a + * later phase reads; pure loop/scratch locals (loop counters, one-shot + * PQExpBuffers, pg_ctl_ret) stay local to whichever phase uses them. + */ +typedef struct SubscriberCreateContext +{ + /* Parsed options */ + char *subscriber_name; + char *base_sub_connstr; + char *base_prov_connstr; + char *replication_sets; + char *databases; + char *postgresql_conf; + char *postgresql_auto_conf; + char *pg_hba_conf; + char *recovery_conf; + int apply_delay; + bool force_text_transfer; + bool stop; + bool drop_slot_if_exists; + char *extra_basebackup_args; + BidirectionalState bidir; + char bidir_manifest_path[MAXPGPATH]; + char bidir_pending_path[MAXPGPATH]; + + /* Resolved database targets */ + char **database_list; + int n_databases; + + /* State set by one phase and consumed by a later one */ + char **slot_names; + char *sub_connstr; + char *prov_connstr; + bool use_existing_data_dir; + RemoteInfo *remote_info; + char *remote_lsn; + CatalogCapture capture; +} SubscriberCreateContext; + +typedef enum { + VERBOSITY_NORMAL, + VERBOSITY_VERBOSE, + VERBOSITY_DEBUG +} VerbosityLevelEnum; + +static char *argv0 = NULL; +static const char *progname; +static char *data_dir = NULL; +static char pid_file[MAXPGPATH]; +static time_t start_time; +static VerbosityLevelEnum verbosity = VERBOSITY_NORMAL; + +/* defined as static so that die() can close them */ +static PGconn *subscriber_conn = NULL; +static PGconn *provider_conn = NULL; + +static void signal_handler(int sig); +static void usage(void); +static void die(const char *fmt,...) +pg_attribute_printf(1, 2); +static void print_msg(VerbosityLevelEnum level, const char *fmt,...) +pg_attribute_printf(2, 3); +static PGresult *debug_exec(PGconn *conn, const char *query); + +static int run_pg_ctl(const char *arg); +static void validate_extra_basebackup_args(const char *args); +static int parse_checked_int(const char *arg, const char *opt_name); +static char *validated_existing_path(const char *arg, const char *file_label); +static void run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args); +static char *reset_subscriber_sysid(const char *data_dir); +static void run_pg_resetwal(const char *data_dir); +static void wait_postmaster_connection(const char *connstr); +static void wait_primary_connection(const char *connstr, int stall_timeout, int max_wait); +static void wait_postmaster_shutdown(void); + +static char *validate_replication_set_input(char *replication_sets); + +static void remove_unwanted_data(PGconn *conn); +static void initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn); +static char *create_restore_point(PGconn *conn, char *restore_point_name); +static char *create_logical_slot(PGconn *conn, const char *slot_name, + bool drop_if_exists, const char *already_exists_hint); +static char *initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists); +static char *create_peer_slot(PGconn *peer_conn, const char *peer_slot_name); +static char *sub_name_for(const char *local_node_name, const char *provider_node_name); +static char *get_origin_name_for_node(PGconn *conn, const char *upstream_node_name, + const char *conn_label, char **sub_name_out); +static void spock_subscribe(PGconn *conn, char *subscriber_name, + char *subscriber_dsn, + char *provider_connstr, + char *replication_sets, + int apply_delay, + bool force_text_transfer); + +static RemoteInfo *get_remote_info(PGconn* conn); + +static bool extension_exists(PGconn *conn, const char *extname); +static void install_extension(PGconn *conn, const char *extname); + +static void ensure_trailing_newline(const char *path); +static void initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args); +static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); +static void check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo); + +static char *read_sysid(const char *data_dir); + +static void WriteRecoveryConf(PQExpBuffer contents); +static void CopyConfFile(char *fromfile, char *tofile, bool append); + +static char *get_connstr_dbname(char *connstr); +static char *get_connstr(char *connstr, char *dbname); +static char *PQconninfoParamsToConnstr(const char *const * keywords, const char *const * values); +static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); + +static bool file_exists(const char *path); +static char *expand_tilde(char *path); +static bool is_pg_dir(const char *path); +static void copy_file(char *fromfile, char *tofile, bool append); +static char *find_other_exec_or_die(const char *argv0, const char *target); +static bool postmaster_is_alive(pid_t pid); +static long get_pgpid(void); +static char **get_database_list(char *databases, int *n_databases); +static char *generate_restore_point_name(void); + +static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out); +static void check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers); +static void check_spock_version_at_least_6(PGconn *conn, const char *node_label); +static void check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes); +static void check_peer_identity(PGconn *peer_conn, const char *expected_name); +static void check_replication_set_equivalence(PGconn *source_conn, + PeerNodeInfo *peers, int num_peers); +static void write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn); +static bool read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out); +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir); +static void stop_postgres_in_data_dir(void); +static bool remove_data_dir_if_forced(bool force); +static bool check_sysid_matches(PGconn *conn, const char *expected_sysid); +static void append_json_string(PQExpBuffer buf, const char *str); + +static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname); +static void check_no_native_subscriptions(PGconn *conn); +static void capture_catalog_state(PGconn *conn, Oid source_nodeid, + CatalogCapture *capture); +static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); +static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); +static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn); +static void create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets); +static char *get_sync_event_lsn(PGconn *conn, const char *node_label); +static char *get_catchup_target_lsn(const char *source_dsn); +static void wait_for_origin_progress(PGconn *conn, const char *origin_name, + const char *target_lsn, const char *watch_sub_name, + const char *context_label, int stall_timeout, int max_wait); +static void wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait); +static void establish_peer_coverage_barrier(BidirectionalState *state, PGconn *n3_conn, + const char *prov_connstr, const char *source_node_name, + const char *source_sub_name, const char *source_slot_name, + const char *subscriber_name, const char *dbname, + const char *base_prov_connstr, + int stall_timeout, int max_wait); +static void clear_forwarding(PGconn *n3_conn, const char *source_dsn, + const char *source_sub_name, const char *source_slot_name, + int stall_timeout, int max_wait); +static void wait_for_sub_replicating(PGconn *conn, const char *sub_name, + int stall_timeout, int max_wait); +static void enable_peer_subs(PGconn *n3_conn, PeerNodeInfo *peers, int num_peers, + int stall_timeout, int max_wait); +static void create_subscription_on_conn(PGconn *conn, const char *sub_name, + const char *provider_dsn, const char *repsets, + const char *conn_label); +static void create_reverse_subscriptions(BidirectionalState *state, const char *subscriber_name, + const char *n3_dsn, const char *replication_sets, + const char *source_dsn, const char *source_node_name, + const char *dbname, const char *base_prov_connstr); +static void wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, int stall_timeout, int max_wait); +static void verify_dataflow_from_n3(PGconn *remote_conn, const char *remote_node_name, + const char *subscriber_name, const char *n3_marker, + int stall_timeout, int max_wait); +static void verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, + const char *remote_marker, const char *slot_name, + const char *sub_name, int stall_timeout, int max_wait); +static void verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *source_sub_name, const char *subscriber_name, + int stall_timeout, int max_wait); +static void set_spock_readonly(PGconn *conn, const char *value); +static Oid get_local_node_id(PGconn *conn); + +static PGconn * +connectdb(const char *connstr) +{ + PGconn *conn; + + conn = PQconnectdb(connstr); + if (PQstatus(conn) != CONNECTION_OK) + die(_("Connection to database failed: %s, connection string was: %s\n"), PQerrorMessage(conn), connstr); + + return conn; +} + +void signal_handler(int sig) +{ + if (sig == SIGINT || sig == SIGTERM) + { + die(_("\nCanceling...\n")); + } +} + +/* + * Append str to buf with JSON string escaping applied, without the + * surrounding quotes (the caller supplies those). Control characters + * below 0x20 are emitted as \uXXXX. jsonapi.h provides a JSON parser but + * no encoder, so this is a small local encoder in the same style as + * src/bin/pg_combinebackup/write_manifest.c. + */ +static void +append_json_string(PQExpBuffer buf, const char *str) +{ + const char *p; + + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': appendPQExpBufferStr(buf, "\\b"); break; + case '\f': appendPQExpBufferStr(buf, "\\f"); break; + case '\n': appendPQExpBufferStr(buf, "\\n"); break; + case '\r': appendPQExpBufferStr(buf, "\\r"); break; + case '\t': appendPQExpBufferStr(buf, "\\t"); break; + case '"': appendPQExpBufferStr(buf, "\\\""); break; + case '\\': appendPQExpBufferStr(buf, "\\\\"); break; + default: + if ((unsigned char) *p < 0x20) + appendPQExpBuffer(buf, "\\u%04x", (unsigned char) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } +} + +/* + * Query the source for all peer nodes in the multi-master cluster. + * Returns the peer count; *peers_out is set to a pg_malloc0'd array. For + * each peer, sub_name is derived as "sub__" + * and slot_name is obtained via spock.spock_gen_slot_name() on the source. + */ +static int +discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out) +{ + static const char *discover_sql = + "SELECT DISTINCT n.node_name, ni.if_dsn" + " FROM spock.subscription s" + " JOIN spock.node n ON s.sub_origin = n.node_id" + " JOIN spock.node_interface ni ON ni.if_id = s.sub_origin_if" + " WHERE n.node_name != $1" + " ORDER BY n.node_name"; + const char *paramValues[3]; + PGresult *res; + PGresult *slot_res; + int npeers; + PeerNodeInfo *peers; + int i; + + paramValues[0] = source_node_name; + print_msg(VERBOSITY_DEBUG, _(" > %s [$1=%s]\n"), discover_sql, source_node_name); + res = PQexecParams(source_conn, discover_sql, + 1, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not discover peer nodes: %s"), + PQerrorMessage(source_conn)); + + npeers = PQntuples(res); + if (npeers == 0) + { + PQclear(res); + die(_("no peer nodes found; source does not appear to be part of a " + "multi-master cluster")); + } + + peers = pg_malloc0(npeers * sizeof(PeerNodeInfo)); + + for (i = 0; i < npeers; i++) + { + peers[i].node_name = pg_strdup(PQgetvalue(res, i, 0)); + peers[i].dsn = pg_strdup(PQgetvalue(res, i, 1)); + peers[i].sub_name = sub_name_for(subscriber_name, peers[i].node_name); + + paramValues[0] = dbname; + paramValues[1] = peers[i].node_name; + paramValues[2] = peers[i].sub_name; + print_msg(VERBOSITY_DEBUG, + _(" > SELECT spock.spock_gen_slot_name($1::name, $2::name, " + "$3::name) [$1=%s, $2=%s, $3=%s]\n"), + dbname, peers[i].node_name, peers[i].sub_name); + slot_res = PQexecParams(source_conn, + "SELECT spock.spock_gen_slot_name" + "($1::name, $2::name, $3::name)", + 3, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(slot_res) != PGRES_TUPLES_OK) + die(_("could not generate slot name for peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(source_conn)); + + peers[i].slot_name = pg_strdup(PQgetvalue(slot_res, 0, 0)); + PQclear(slot_res); + + print_msg(VERBOSITY_VERBOSE, + _(" discovered peer: %s (slot: %s)\n"), + peers[i].node_name, peers[i].slot_name); + } + + PQclear(res); + *peers_out = peers; + return npeers; +} + +/* + * Verify Spock version on conn (the source or a peer): an old apply + * worker would advance the wrong-named origin, so this must be checked + * everywhere up front, not just on the source. + */ +static void +check_spock_version_at_least_6(PGconn *conn, const char *node_label) +{ + PGresult *res; + + res = debug_exec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not query Spock extension version on \"%s\": %s"), + node_label, PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("Spock extension is not installed on \"%s\""), node_label); + } + { + const char *ver = PQgetvalue(res, 0, 0); + int major = 0; + + /* + * die() exits immediately -- ver points inside res, so it must + * not be PQclear()'d first (that would be a use-after-free when + * die()'s own formatting reads ver). + */ + if (sscanf(ver, "%d.", &major) < 1) + die(_("could not parse Spock version \"%s\" on \"%s\""), ver, node_label); + if (major < 6) + die(_("Spock version %s on \"%s\" is too old for bidirectional " + "join; require >= 6.0.0"), ver, node_label); + } + PQclear(res); +} + +/* + * Validate the actual directed subscription graph from one node's own + * catalog, not just a count: exactly one healthy (status = 'replicating') + * subscription from every other node in the set, no self-reference, no + * edge from outside the set, and no duplicate edge from the same origin + * regardless of status. + */ +static void +check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes) +{ + PGresult *res; + bool *healthy; + int *edge_count; + int i; + + /* + * spock.sub_show_status() (not raw sub_enabled) so "enabled" also + * means "actually replicating" -- a worker that's down or still + * initializing must not satisfy the mesh. + */ + res = debug_exec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check subscription topology on \"%s\": %s"), + this_node_name, PQerrorMessage(conn)); + } + + healthy = pg_malloc0(total_nodes * sizeof(bool)); + edge_count = pg_malloc0(total_nodes * sizeof(int)); + + for (i = 0; i < PQntuples(res); i++) + { + const char *origin_name = PQgetvalue(res, i, 0); + const char *status = PQgetvalue(res, i, 1); + int idx = -1; + int k; + + if (strcmp(origin_name, this_node_name) == 0) + die(_("node \"%s\" has a subscription whose origin is itself; " + "corrupt or misconfigured topology"), this_node_name); + + for (k = 0; k < total_nodes; k++) + { + if (strcmp(all_names[k], origin_name) == 0) + { + idx = k; + break; + } + } + if (idx == -1) + die(_("node \"%s\" has a subscription from \"%s\", which is not " + "part of the discovered node set; partial-mesh or " + "unknown-node topologies are not supported"), + this_node_name, origin_name); + + /* + * Count regardless of status: an extra disabled duplicate from + * the same origin is still a duplicate edge. + */ + edge_count[idx]++; + if (edge_count[idx] > 1) + die(_("node \"%s\" has more than one subscription from \"%s\" " + "(status \"%s\"); duplicate edges are not supported"), + this_node_name, origin_name, status); + + if (strcmp(status, "replicating") == 0) + healthy[idx] = true; + } + PQclear(res); + + for (i = 0; i < total_nodes; i++) + { + if (strcmp(all_names[i], this_node_name) == 0) + continue; /* skip self */ + if (!healthy[i]) + { + pg_free(healthy); + pg_free(edge_count); + die(_("node \"%s\" has no healthy (status = 'replicating') " + "subscription from \"%s\"; full-mesh topology of live " + "replication is required for bidirectional join"), + this_node_name, all_names[i]); + } + } + pg_free(healthy); + pg_free(edge_count); +} + +/* + * Confirm the peer identifies itself as the name it was discovered + * under, so a node-name collision or wrong DSN can't silently validate + * the mesh against the wrong node. + */ +static void +check_peer_identity(PGconn *peer_conn, const char *expected_name) +{ + PGresult *res; + + res = debug_exec(peer_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify identity of peer \"%s\": %s"), + expected_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), expected_name) != 0) + { + char *actual_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("peer discovered as \"%s\" identifies itself as \"%s\" once " + "connected; node-name/identity mismatch, refusing to trust " + "this topology\n"), expected_name, actual_name); + } + PQclear(res); +} + +/* + * Build a canonical, comparable fingerprint of one replication set as + * defined by its owning node: operation flags, each member table (sorted, + * with column list, row filter, and schema), then each member sequence + * (sorted). Scoped by node_id since spock.replication_set is keyed + * UNIQUE(set_nodeid, set_name) -- a set replicated via DDL becomes the + * replaying node's own row, not an echo. selected_filter restricts this + * to sets actually referenced by a subscription's sub_replication_sets, + * since unused/scratch repsets can legitimately differ between nodes. + */ +typedef struct RepsetFingerprintEntry +{ + char *set_name; + char *fingerprint; +} RepsetFingerprintEntry; + +static void +compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filter, + RepsetFingerprintEntry **out, int *nout) +{ + PGresult *res; + RepsetFingerprintEntry *entries; + int n; + int i; + PQExpBuffer query = createPQExpBuffer(); + + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_nodeid = %u" + " AND (%s)" + " ORDER BY set_name", node_id, selected_filter); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + destroyPQExpBuffer(query); + die(_("could not fingerprint replication sets: %s\n"), PQerrorMessage(conn)); + } + + n = PQntuples(res); + entries = pg_malloc0(n * sizeof(RepsetFingerprintEntry)); + + for (i = 0; i < n; i++) + { + PQExpBuffer fp = createPQExpBuffer(); + PGresult *tres; + PGresult *sres; + int j; + + entries[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + appendPQExpBuffer(fp, "flags=%s%s%s%s;", + PQgetvalue(res, i, 1)[0] == 't' ? "i" : "", + PQgetvalue(res, i, 2)[0] == 't' ? "u" : "", + PQgetvalue(res, i, 3)[0] == 't' ? "d" : "", + PQgetvalue(res, i, 4)[0] == 't' ? "t" : ""); + + printfPQExpBuffer(query, + "SELECT rts.set_reloid::regclass::text, rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rts.set_reloid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + tres = debug_exec(conn, query->data); + if (PQresultStatus(tres) != PGRES_TUPLES_OK) + { + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint table memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + + for (j = 0; j < PQntuples(tres); j++) + { + const char *qualified_table = PQgetvalue(tres, j, 0); + PGresult *cres; + PQExpBuffer schema_query = createPQExpBuffer(); + int k; + + appendPQExpBuffer(fp, "tbl=%s|cols=%s|filter=%s|schema=(", + qualified_table, + PQgetisnull(tres, j, 1) ? "*" : PQgetvalue(tres, j, 1), + PQgetisnull(tres, j, 2) ? "-" : PQgetvalue(tres, j, 2)); + + /* + * Schema fingerprint: relation kind and replica identity, + * then per-column name, type, typmod (varchar(10) vs + * varchar(100) is otherwise invisible), collation, + * nullability, and generated/identity status -- so a + * divergent column or relation definition is caught even if + * repset membership itself matches. + */ + printfPQExpBuffer(schema_query, + "SELECT relkind::text, relreplident::text" + " FROM pg_class WHERE oid = %s::regclass", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = debug_exec(conn, schema_query->data); + if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(schema_query); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint relation kind of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + appendPQExpBuffer(fp, "relkind=%s|replident=%s|", + PQgetvalue(cres, 0, 0), PQgetvalue(cres, 0, 1)); + PQclear(cres); + + printfPQExpBuffer(schema_query, + "SELECT a.attname, a.atttypid::regtype::text, a.atttypmod," + " a.attnotnull, a.attidentity, a.attgenerated," + " COALESCE(co.collname, '')" + " FROM pg_attribute a" + " LEFT JOIN pg_collation co ON co.oid = a.attcollation" + " WHERE a.attrelid = %s::regclass AND a.attnum > 0" + " AND NOT a.attisdropped ORDER BY a.attnum", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = debug_exec(conn, schema_query->data); + destroyPQExpBuffer(schema_query); + if (PQresultStatus(cres) != PGRES_TUPLES_OK) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint schema of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + for (k = 0; k < PQntuples(cres); k++) + appendPQExpBuffer(fp, "%s%s:%s:%s:notnull=%s:ident=%s:gen=%s:coll=%s", + k > 0 ? "," : "", + PQgetvalue(cres, k, 0), + PQgetvalue(cres, k, 1), + PQgetvalue(cres, k, 2), + PQgetvalue(cres, k, 3), + PQgetvalue(cres, k, 4), + PQgetvalue(cres, k, 5), + PQgetvalue(cres, k, 6)); + appendPQExpBufferStr(fp, ");"); + PQclear(cres); + } + PQclear(tres); + + printfPQExpBuffer(query, + "SELECT rss.set_seqoid::regclass::text" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rss.set_seqoid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + sres = debug_exec(conn, query->data); + if (PQresultStatus(sres) != PGRES_TUPLES_OK) + { + PQclear(sres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint sequence memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + for (j = 0; j < PQntuples(sres); j++) + appendPQExpBuffer(fp, "seq=%s;", PQgetvalue(sres, j, 0)); + PQclear(sres); + + entries[i].fingerprint = pg_strdup(fp->data); + destroyPQExpBuffer(fp); + } + PQclear(res); + destroyPQExpBuffer(query); + + *out = entries; + *nout = n; +} + +static void +free_repset_fingerprints(RepsetFingerprintEntry *entries, int n) +{ + int i; + + for (i = 0; i < n; i++) + { + pg_free(entries[i].set_name); + pg_free(entries[i].fingerprint); + } + pg_free(entries); +} + +/* + * Return a comma-separated list of every replication set actually + * referenced by conn's own subscriptions (sub_replication_sets), rather + * than every set that happens to exist locally. A --bidirectional join + * uses this to make the joining node inherit the sets already in use by + * the cluster it's joining, instead of accepting a separately specified + * list that could diverge from what check_replication_set_equivalence() + * (just below, via the identical query) validates. Caller frees the + * result. + */ +static char * +get_source_mesh_replication_sets(PGconn *conn) +{ + PGresult *res; + PQExpBuffer list; + char *result; + int i; + + res = debug_exec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine the cluster's replication sets: %s\n"), + PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "determine which replication sets to use\n")); + } + + list = createPQExpBuffer(); + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "%s%s", i > 0 ? "," : "", PQgetvalue(res, i, 0)); + PQclear(res); + + result = pg_strdup(list->data); + destroyPQExpBuffer(list); + return result; +} + +/* + * Build a SQL boolean expression ("set_name IN (...)") over the union of + * every replication set actually referenced by conn's own subscriptions + * (sub_replication_sets), rather than every set that happens to exist + * locally. Caller frees the result. + */ +static char * +build_selected_set_name_filter(PGconn *conn) +{ + PGresult *res; + PQExpBuffer filter; + char *result; + int i; + + res = debug_exec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine selected replication sets: %s\n"), PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "verify replication-set equivalence\n")); + } + + filter = createPQExpBuffer(); + appendPQExpBufferStr(filter, "set_name IN ("); + for (i = 0; i < PQntuples(res); i++) + { + char *name = PQgetvalue(res, i, 0); + + appendPQExpBuffer(filter, "%s%s", i > 0 ? ", " : "", + PQescapeLiteral(conn, name, strlen(name))); + } + appendPQExpBufferStr(filter, ")"); + PQclear(res); + + result = pg_strdup(filter->data); + destroyPQExpBuffer(filter); + return result; +} + +/* + * The forwarding path (peer -> source -> n3) and the future direct path + * (peer -> n3) must select exactly the same changes, or a change omitted + * on one path is lost once the direct subscription takes over. Compare + * every selected replication set's fingerprint between the source and + * each peer; reject any mismatch or missing/extra set on either side. + */ +static void +check_replication_set_equivalence(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +{ + Oid source_nodeid = get_local_node_id(source_conn); + char *selected_filter = build_selected_set_name_filter(source_conn); + RepsetFingerprintEntry *source_fps; + int num_source_fps; + int i; + + compute_repset_fingerprints(source_conn, source_nodeid, selected_filter, + &source_fps, &num_source_fps); + + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + Oid peer_nodeid; + RepsetFingerprintEntry *peer_fps; + int num_peer_fps; + int j; + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + peer_nodeid = get_local_node_id(peer_conn); + compute_repset_fingerprints(peer_conn, peer_nodeid, selected_filter, + &peer_fps, &num_peer_fps); + + /* + * die() exits immediately, so none of the branches below free + * source_fps/peer_fps before calling it -- freeing first and then + * still reading source_fps[j]/peer_fps[j] in the same die() call's + * arguments would be a use-after-free (the process is about to + * exit anyway; nothing else in this file frees before die() either). + */ + for (j = 0; j < num_source_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_peer_fps; k++) + { + if (strcmp(source_fps[j].set_name, peer_fps[k].set_name) != 0) + continue; + found = true; + if (strcmp(source_fps[j].fingerprint, peer_fps[k].fingerprint) != 0) + die(_("replication set \"%s\" differs between the source " + "and peer \"%s\" (membership, flags, columns, row " + "filter, or schema) -- the forwarding path and a " + "future direct peer subscription would not select " + "the same changes, risking permanently lost data " + "on cutover. Reconcile the definitions before " + "retrying.\n"), + source_fps[j].set_name, peers[i].node_name); + break; + } + if (!found) + die(_("replication set \"%s\" exists on the source but not " + "on peer \"%s\"\n"), source_fps[j].set_name, peers[i].node_name); + } + for (j = 0; j < num_peer_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_source_fps; k++) + if (strcmp(peer_fps[j].set_name, source_fps[k].set_name) == 0) + { + found = true; + break; + } + if (!found) + die(_("replication set \"%s\" exists on peer \"%s\" but not " + "on the source\n"), peer_fps[j].set_name, peers[i].node_name); + } + + free_repset_fingerprints(peer_fps, num_peer_fps); + PQfinish(peer_conn); + } + + free_repset_fingerprints(source_fps, num_source_fps); + pg_free(selected_filter); +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0 on every node, track_commit_timestamp + * on, no pending DDL, an actual full-mesh subscription graph (not just a + * count), replication-set/schema equivalence across the source and every + * peer, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + int total_nodes = num_peers + 1; + char **all_names = pg_malloc(total_nodes * sizeof(char *)); + + all_names[0] = pg_strdup(source_node_name); + for (i = 0; i < num_peers; i++) + all_names[i + 1] = pg_strdup(peers[i].node_name); + + check_spock_version_at_least_6(source_conn, "source"); + + /* track_commit_timestamp must be on at the source */ + res = debug_exec(source_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check track_commit_timestamp: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + die(_("track_commit_timestamp must be on for bidirectional join (source)")); + PQclear(res); + + /* + * All outbound replication caught up to the source's current WAL + * position -- i.e. nothing (DDL or data) still in flight to an + * existing peer. spock.queue's row count is not a usable signal here: + * queue_message() (spock_queue.c) only ever inserts into it, so its + * count is monotonically non-decreasing and is never zero on any node + * that has replicated so much as a single DDL statement. + */ + res = debug_exec(source_conn, + "SELECT COUNT(*) FROM pg_replication_slots" + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" + " AND (confirmed_flush_lsn IS NULL" + " OR confirmed_flush_lsn < pg_current_wal_lsn())"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check replication slot lag: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + die(_("source has unreplicated changes pending to an existing peer; " + "wait for replication to drain before joining")); + PQclear(res); + + /* Full-mesh directed-graph check, from the source's own perspective. */ + check_mesh_edges(source_conn, source_node_name, all_names, total_nodes); + + /* + * Per-peer: connectivity, Spock version, track_commit_timestamp, and + * the full-mesh directed-graph check from each peer's own perspective + * (a mesh that's only complete as seen from the source is not a mesh). + */ + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + + print_msg(VERBOSITY_VERBOSE, + _(" checking peer %s ...\n"), peers[i].node_name); + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + check_peer_identity(peer_conn, peers[i].node_name); + check_spock_version_at_least_6(peer_conn, peers[i].node_name); + + res = debug_exec(peer_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + /* + * die() exits immediately -- PQerrorMessage() needs peer_conn + * still open, so PQfinish() must not run first (that would be + * a use-after-free when die()'s own formatting reads it). + */ + PQclear(res); + die(_("could not check track_commit_timestamp on peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + { + PQclear(res); + PQfinish(peer_conn); + die(_("track_commit_timestamp must be on for bidirectional join " + "(peer \"%s\")"), peers[i].node_name); + } + PQclear(res); + + check_mesh_edges(peer_conn, peers[i].node_name, all_names, total_nodes); + + PQfinish(peer_conn); + } + + /* Replication-set & schema equivalence: run once the mesh is sound. */ + check_replication_set_equivalence(source_conn, peers, num_peers); + + for (i = 0; i < total_nodes; i++) + pg_free(all_names[i]); + pg_free(all_names); + + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); +} + +/* + * The physical-backup path runs once per data directory, so it requires + * exactly one spock-configured database on the source instance. Checked + * against actual spock configuration, not --databases/--provider-dsn, + * since the instance can host other unrelated databases. Fails closed: + * any database we cannot inspect aborts the run rather than being + * treated as spock-free. datallowconn is not used to skip databases -- + * a database with connections disabled can still hold spock catalog + * state -- only true templates are excluded. + */ +static void +check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname) +{ + PGresult *res; + int i; + PQExpBuffer others = createPQExpBuffer(); + int other_count = 0; + + res = debug_exec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not list databases on source: %s\n"), PQerrorMessage(conn)); + } + + for (i = 0; i < PQntuples(res); i++) + { + char *dbname = PQgetvalue(res, i, 0); + char *db_connstr; + PGconn *db_conn; + PGresult *ext_res; + PGresult *node_res; + + if (strcmp(dbname, current_dbname) == 0) + continue; + + db_connstr = get_connstr((char *) base_prov_connstr, dbname); + db_conn = PQconnectdb(db_connstr); + if (PQstatus(db_conn) != CONNECTION_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not connect to " + "\"%s\" to check: %s\n"), dbname, errmsg); + } + + ext_res = debug_exec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not query \"%s\": " + "%s\n"), dbname, errmsg); + } + + if (PQntuples(ext_res) > 0) + { + node_res = debug_exec(db_conn, "SELECT 1 FROM spock.local_node"); + if (PQresultStatus(node_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(node_res); + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on " + "the source has spock configured, but could not query " + "spock.local_node in \"%s\": %s\n"), dbname, errmsg); + } + + if (PQntuples(node_res) > 0) + { + appendPQExpBuffer(others, "%s%s", other_count ? ", " : "", dbname); + other_count++; + } + PQclear(node_res); + } + PQclear(ext_res); + PQfinish(db_conn); + } + PQclear(res); + + if (other_count > 0) + die(_("--bidirectional requires exactly one spock-configured database " + "on the source instance; also found spock configured on: %s\n"), + others->data); + + destroyPQExpBuffer(others); +} + +/* + * A physical base backup copies native (non-spock) logical subscriptions + * too, which DROP EXTENSION spock doesn't touch. Once n3 is promoted and + * restarted, an enabled native subscription would start consuming from + * its provider as a second, unintended consumer. pg_subscription is a + * shared catalog, so one query sees every database's rows. + */ +static void +check_no_native_subscriptions(PGconn *conn) +{ + PGresult *res; + + res = debug_exec(conn, + "SELECT s.subname, d.datname" + " FROM pg_subscription s" + " JOIN pg_database d ON d.oid = s.subdbid" + " WHERE s.subenabled"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check for native logical subscriptions on the " + "source instance: %s\n"), PQerrorMessage(conn)); + } + + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; + + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s (database %s)", + PQgetvalue(res, i, 0), PQgetvalue(res, i, 1)); + + PQclear(res); + die(_("--bidirectional requires no enabled native (non-spock) logical " + "subscriptions anywhere on the source instance -- a physical " + "backup would copy them, and they would start consuming on " + "the new node as an unintended second consumer once " + "promoted: %s\nDisable or drop these subscriptions before " + "retrying.\n"), list->data); + } + PQclear(res); +} + +/* + * Manifest field names, shared between serialize_manifest() and the + * manifest_* parser callbacks below, so a typo in one becomes a + * compile error instead of a silent read/write mismatch. + */ +#define MF_SUBSCRIBER_NAME "subscriber_name" +#define MF_DBNAME "dbname" +#define MF_SOURCE_DSN "source_dsn" +#define MF_SOURCE_SLOT_NAME "source_slot_name" +#define MF_SOURCE_RESTORE_LSN "source_restore_lsn" +#define MF_NODE_DSN "node_dsn" +#define MF_NODE_SYSID "node_sysid" +#define MF_SOURCE_REVERSE_SUB_CREATED "source_reverse_sub_created" +#define MF_PEERS "peers" +#define MF_NODE_NAME "node_name" +#define MF_PEER_DSN "peer_dsn" +#define MF_SUB_NAME_ON_N3 "sub_name_on_n3" +#define MF_PEER_SLOT_NAME "peer_slot_name" +#define MF_SLOT_CREATION_LSN "slot_creation_lsn" +#define MF_REVERSE_SUB_CREATED "reverse_sub_created" + +/* + * Append one """: """ field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). A NULL value serializes as an empty string, matching + * append_json_string()'s existing null-becomes-empty convention here. + */ +static void +append_json_str_field(PQExpBuffer buf, const char *indent, const char *field_name, + const char *value, bool trailing_comma) +{ + appendPQExpBuffer(buf, "%s\"%s\": \"", indent, field_name); + if (value) + append_json_string(buf, value); + appendPQExpBufferStr(buf, trailing_comma ? "\",\n" : "\"\n"); +} + +/* + * Append one """: true|false" field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). + */ +static void +append_json_bool_field(PQExpBuffer buf, const char *indent, const char *field_name, + bool value, bool trailing_comma) +{ + appendPQExpBuffer(buf, "%s\"%s\": %s%s\n", indent, field_name, + value ? "true" : "false", trailing_comma ? "," : ""); +} + +/* + * Hand-serialize the bidirectional state manifest schema into buf, as + * JSON, with string values escaped by append_json_string(). + */ +static void +serialize_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, PQExpBuffer buf) +{ + int i; + + appendPQExpBufferStr(buf, "{\n"); + appendPQExpBufferStr(buf, " \"version\": 1,\n"); + + append_json_str_field(buf, " ", MF_SUBSCRIBER_NAME, subscriber_name, true); + append_json_str_field(buf, " ", MF_DBNAME, dbname, true); + append_json_str_field(buf, " ", MF_SOURCE_DSN, source_dsn, true); + append_json_str_field(buf, " ", MF_SOURCE_SLOT_NAME, state->source_slot_name, true); + append_json_str_field(buf, " ", MF_SOURCE_RESTORE_LSN, state->source_restore_lsn, true); + append_json_str_field(buf, " ", MF_NODE_DSN, state->node_dsn, true); + append_json_str_field(buf, " ", MF_NODE_SYSID, state->node_sysid, true); + append_json_bool_field(buf, " ", MF_SOURCE_REVERSE_SUB_CREATED, + state->source_reverse_sub_created, true); + + appendPQExpBuffer(buf, " \"%s\": [\n", MF_PEERS); + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *p = &state->peers[i]; + bool last = (i == state->num_peers - 1); + + appendPQExpBufferStr(buf, " {\n"); + append_json_str_field(buf, " ", MF_NODE_NAME, p->node_name, true); + append_json_str_field(buf, " ", MF_PEER_DSN, p->dsn, true); + append_json_str_field(buf, " ", MF_SUB_NAME_ON_N3, p->sub_name, true); + append_json_str_field(buf, " ", MF_PEER_SLOT_NAME, p->slot_name, true); + append_json_str_field(buf, " ", MF_SLOT_CREATION_LSN, p->slot_creation_lsn, true); + append_json_bool_field(buf, " ", MF_REVERSE_SUB_CREATED, + p->reverse_sub_created, false); + appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); + } + appendPQExpBufferStr(buf, " ]\n"); + appendPQExpBufferStr(buf, "}\n"); +} + +/* + * Atomically replace path's contents with the len bytes at data: write + * to a ".tmp" sibling with mode 0600 up front (never a post-hoc + * chmod, since the caller's data can embed a password), fsync, rename + * over path, then fsync the containing directory -- a crash right + * after this returns must not lose the write (e.g. the only cleanup + * record for a resource created just before it). O_EXCL|O_NOFOLLOW + * refuses to write through a pre-existing file or planted symlink, + * except a leftover .tmp from a previous crashed run. + */ +static void +durably_replace_manifest(const char *path, const char *data, size_t len) +{ + char tmp_path[MAXPGPATH]; + int fd; + ssize_t written; + + snprintf(tmp_path, MAXPGPATH, "%s.tmp", path); + + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + if (fd < 0 && errno == EEXIST) + { + if (unlink(tmp_path) != 0) + die(_("could not remove stale manifest temp file \"%s\": %s"), + tmp_path, strerror(errno)); + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + } + if (fd < 0) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + written = write(fd, data, len); + if (written < 0 || (size_t) written != len) + { + close(fd); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + + if (fsync(fd) != 0) + { + close(fd); + unlink(tmp_path); + die(_("could not fsync manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (close(fd) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + path, strerror(errno)); + + /* + * fsync_parent_path() already treats "filesystem doesn't support + * directory fsync" as success internally, so a nonzero return here + * is a genuine failure that can orphan the resource(s) this + * manifest is the only record of, after a crash. Fatal, like the + * durability steps above. + */ + if (fsync_parent_path(path) != 0) + die(_("could not fsync directory containing \"%s\": %s\n"), + path, strerror(errno)); +} + +/* + * Write the bidirectional state manifest to state->manifest_path. + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + + serialize_manifest(state, subscriber_name, dbname, source_dsn, buf); + durably_replace_manifest(state->manifest_path, buf->data, buf->len); + + destroyPQExpBuffer(buf); +} + +/* + * Semantic-action state for read_manifest(). Passed as void *semstate to all + * pg_parse_json callbacks; tracks nesting depth and accumulates field values. + */ +typedef struct ManifestParseState +{ + /* outputs written by scalar callback */ + char **subscriber_name_out; + char **dbname_out; + char **source_dsn_out; + BidirectionalState *bidir; + + /* parser context */ + int depth; /* object/array nesting depth */ + bool in_peers; /* inside the top-level "peers" array */ + bool in_peer_obj; /* inside one peer object */ + char *cur_field; /* current object field name (owned by us) */ + + /* per-peer accumulator, flushed on each object_end inside peers */ + char *peer_node_name; + char *peer_dsn; + char *peer_sub_name; + char *peer_slot_name; + char *peer_slot_creation_lsn; + bool peer_reverse_sub_created; + int peer_capacity; +} ManifestParseState; + +static JsonParseErrorType +manifest_object_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->in_peers && s->depth == 3) + s->in_peer_obj = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_object_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peer_obj && s->depth == 3) + { + int i = s->bidir->num_peers; + + if (i >= s->peer_capacity) + { + s->peer_capacity = (s->peer_capacity > 0) ? s->peer_capacity * 2 : 4; + s->bidir->peers = pg_realloc(s->bidir->peers, + s->peer_capacity * sizeof(PeerNodeInfo)); + } + s->bidir->peers[i].node_name = s->peer_node_name; + s->bidir->peers[i].dsn = s->peer_dsn; + s->bidir->peers[i].sub_name = s->peer_sub_name; + s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->peers[i].slot_creation_lsn = s->peer_slot_creation_lsn; + s->bidir->peers[i].reverse_sub_created = s->peer_reverse_sub_created; + s->bidir->num_peers++; + s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->peer_slot_creation_lsn = NULL; + s->peer_reverse_sub_created = false; + s->in_peer_obj = false; + } + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->depth == 2 && s->cur_field != NULL && + strcmp(s->cur_field, MF_PEERS) == 0) + s->in_peers = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peers && s->depth == 2) + s->in_peers = false; + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_ofield_start(void *st, char *fname, bool isnull) +{ + ManifestParseState *s = (ManifestParseState *) st; + + (void) isnull; + pg_free(s->cur_field); + s->cur_field = pg_strdup(fname); + pg_free(fname); /* callback owns the token */ + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_scalar(void *st, char *token, JsonTokenType tokentype) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->cur_field == NULL) + { + pg_free(token); + return JSON_SUCCESS; + } + + /* + * Per-peer creation-state flags are JSON booleans, not strings -- + * handle them before the string-only fields below (which free and + * ignore anything that isn't JSON_TOKEN_STRING). + */ + if (s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, MF_REVERSE_SUB_CREATED) == 0) + s->peer_reverse_sub_created = value; + pg_free(token); + return JSON_SUCCESS; + } + + /* Top-level creation-state flags are also JSON booleans. */ + if (!s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, MF_SOURCE_REVERSE_SUB_CREATED) == 0) + s->bidir->source_reverse_sub_created = value; + pg_free(token); + return JSON_SUCCESS; + } + + if (tokentype != JSON_TOKEN_STRING) + { + pg_free(token); + return JSON_SUCCESS; + } + + if (!s->in_peer_obj) + { + /* top-level scalar fields */ + if (strcmp(s->cur_field, MF_SUBSCRIBER_NAME) == 0) + *s->subscriber_name_out = token; + else if (strcmp(s->cur_field, MF_DBNAME) == 0) + *s->dbname_out = token; + else if (strcmp(s->cur_field, MF_SOURCE_DSN) == 0) + *s->source_dsn_out = token; + else if (strcmp(s->cur_field, MF_SOURCE_SLOT_NAME) == 0) + s->bidir->source_slot_name = token; + else if (strcmp(s->cur_field, MF_NODE_DSN) == 0) + s->bidir->node_dsn = token; + else if (strcmp(s->cur_field, MF_NODE_SYSID) == 0) + s->bidir->node_sysid = token; + else + pg_free(token); + } + else + { + /* per-peer scalar fields */ + if (strcmp(s->cur_field, MF_NODE_NAME) == 0) + s->peer_node_name = token; + else if (strcmp(s->cur_field, MF_PEER_DSN) == 0) + s->peer_dsn = token; + else if (strcmp(s->cur_field, MF_SUB_NAME_ON_N3) == 0) + s->peer_sub_name = token; + else if (strcmp(s->cur_field, MF_PEER_SLOT_NAME) == 0) + s->peer_slot_name = token; + else if (strcmp(s->cur_field, MF_SLOT_CREATION_LSN) == 0) + s->peer_slot_creation_lsn = token; + else + pg_free(token); + } + return JSON_SUCCESS; +} + +/* + * Read the bidirectional manifest from manifest_path. Returns false if + * the file does not exist (nothing to clean up); dies if it exists but + * cannot be read or is malformed. On success, sets *subscriber_name_out, + * *dbname_out, *source_dsn_out, and populates state->peers[]. + * + * Uses pg_parse_json (common/jsonapi.h) for JSON lexing, so string + * quoting, escape sequences, and nesting are handled correctly. + */ +static bool +read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out) +{ + struct stat st; + char *content; + FILE *f; + JsonLexContext *lex; + JsonSemAction sem; + ManifestParseState pstate; + JsonParseErrorType result; + + if (stat(manifest_path, &st) != 0) + return false; + + content = pg_malloc(st.st_size + 1); + f = fopen(manifest_path, "r"); + if (f == NULL) + die(_("could not open manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + + if ((size_t) fread(content, 1, st.st_size, f) != (size_t) st.st_size) + { + fclose(f); + die(_("could not read manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + } + content[st.st_size] = '\0'; + fclose(f); + + memset(&pstate, 0, sizeof(pstate)); + pstate.subscriber_name_out = subscriber_name_out; + pstate.dbname_out = dbname_out; + pstate.source_dsn_out = source_dsn_out; + pstate.bidir = state; + + memset(&sem, 0, sizeof(sem)); + sem.semstate = &pstate; + sem.object_start = manifest_object_start; + sem.object_end = manifest_object_end; + sem.array_start = manifest_array_start; + sem.array_end = manifest_array_end; + sem.object_field_start = manifest_ofield_start; + sem.scalar = manifest_scalar; + + lex = makeJsonLexContextCstringLen(NULL, content, st.st_size, + PG_UTF8, true); + result = pg_parse_json(lex, &sem); + pg_free(content); + pg_free(pstate.cur_field); + + if (result != JSON_SUCCESS) + { + char *detail = json_errdetail(result, lex); + + freeJsonLexContext(lex); + die(_("manifest file \"%s\" is malformed: %s"), manifest_path, detail); + } + freeJsonLexContext(lex); + + if (!*subscriber_name_out || !*dbname_out || !*source_dsn_out) + die(_("manifest file \"%s\" is malformed or missing required fields"), + manifest_path); + + return true; +} + +/* + * If data_dir holds a running postmaster, stop it (fast mode) and wait + * for shutdown. No-op if data_dir is unset, doesn't exist, or has no + * postmaster.pid. + */ +static void +stop_postgres_in_data_dir(void) +{ + struct stat st; + + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s ...\n"), data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } +} + +/* + * If data_dir exists, remove it when force is true (stopping postgres in + * it first, defensively, in case the caller hasn't already); if force is + * false, leave it in place with a hint. Returns false only when removal + * was attempted and actually failed; a missing data_dir, an unset one, + * or force being false are all "nothing to report" and return true. + */ +static bool +remove_data_dir_if_forced(bool force) +{ + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return true; + + if (!force) + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + return true; + } + + stop_postgres_in_data_dir(); + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + return false; + } + + return true; +} + +/* + * Check whether conn's system_identifier (from pg_control_system()) + * matches expected_sysid. Any failure to confirm -- query error, no + * row, or an outright mismatch -- returns false. + */ +static bool +check_sysid_matches(PGconn *conn, const char *expected_sysid) +{ + PGresult *res; + bool matches; + + res = debug_exec(conn, "SELECT system_identifier FROM pg_control_system()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + return false; + } + matches = strcmp(PQgetvalue(res, 0, 0), expected_sysid) == 0; + PQclear(res); + return matches; +} + +/* + * Outcome of an idempotent drop-if-exists cleanup operation. A plain bool + * can't tell "nothing to do" apart from "confirmed gone", which callers + * that gate further action (or --cleanup's own retry decision) on actual + * removal need to distinguish from a query failure. + */ +typedef enum +{ + CLEANUP_DROP_FAILED, /* query failed; removal not confirmed */ + CLEANUP_DROP_ABSENT, /* query succeeded; resource did not exist */ + CLEANUP_DROP_REMOVED /* query succeeded; resource was dropped */ +} CleanupDropOutcome; + +/* + * Drop the replication slot named slot_name on conn if it exists. + * node_label identifies the node in progress/warning messages. + */ +static CleanupDropOutcome +drop_logical_slot_if_exists(PGconn *conn, const char *slot_name, const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + slot_name); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop slot %s on %s: %s\n"), + slot_name, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (PQntuples(res) > 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped slot %s on %s\n"), slot_name, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop the reverse subscription named "sub__" + * on conn, via spock.sub_drop(..., ifexists := true) -- an absent + * subscription is not an error. node_label identifies the node in + * progress/warning messages. spock.sub_drop() itself returns whether it + * found and dropped a subscription, which is read back here to distinguish + * CLEANUP_DROP_ABSENT from CLEANUP_DROP_REMOVED; CLEANUP_DROP_FAILED (and a + * warning) is returned only on an actual query failure. spock.sub_drop() + * is declared RETURNS oid in the catalog (sql/spock--6.0.0.sql) even + * though its C implementation (spock_drop_subscription) returns a bool + * Datum, so the wire text is "0"/"1", not "f"/"t" -- compare against "0" + * rather than checking for boolean-formatted text. + */ +static CleanupDropOutcome +drop_reverse_sub(PGconn *conn, const char *node_name, const char *subscriber_name, + const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *reverse_sub = sub_name_for(node_name, subscriber_name); + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(conn, reverse_sub, strlen(reverse_sub))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on %s: %s\n"), + reverse_sub, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped reverse subscription %s on %s\n"), + reverse_sub, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + pg_free(reverse_sub); + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop every subscription this run may have created on n3 itself: the + * catchup subscription to the source and any disabled peer subscriptions. + * A freshly-provisioned n3 has no other legitimate spock.subscription + * rows, so it's safe to drop everything found -- but only once node_sysid + * confirms node_dsn still reaches that same n3, since a manifest can + * outlive the node it describes (DNS change, load balancer, reused port). + * node_dsn is only set once node_create() has run, so its absence just + * means there's nothing on n3 yet, and this returns true without doing + * anything. spock.sub_drop() on n3 kills that subscription's local apply + * worker and drops the matching remote slot on its origin itself, so n3 + * never needs to be stopped just to release a slot it holds open + * elsewhere. + */ +static bool +cleanup_verified_subscriber_node(BidirectionalState *state, const char *subscriber_name) +{ + PGconn *n3_conn; + PGresult *res; + PQExpBuffer query; + int i; + bool fully_cleaned = true; + + if (!state->node_dsn || !state->node_dsn[0]) + return true; + + n3_conn = PQconnectdb(state->node_dsn); + if (PQstatus(n3_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to subscriber \"%s\"; its " + "subscription(s) may still exist: %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + PQfinish(n3_conn); + return false; + } + + if (!state->node_sysid || !state->node_sysid[0] || + !check_sysid_matches(n3_conn, state->node_sysid)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: node_dsn for subscriber \"%s\" cannot be " + "confirmed to still identify the node this run " + "created (missing or mismatched system identifier); " + "refusing to drop subscriptions there. Investigate " + "manually.\n"), subscriber_name); + PQfinish(n3_conn); + return false; + } + + query = createPQExpBuffer(); + res = debug_exec(n3_conn, "SELECT sub_name FROM spock.subscription"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + for (i = 0; i < PQntuples(res); i++) + { + char *sub_name = PQgetvalue(res, i, 0); + PGresult *drop_res; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(n3_conn, sub_name, strlen(sub_name))); + drop_res = debug_exec(n3_conn, query->data); + if (PQresultStatus(drop_res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_NORMAL, + _(" dropped subscriber subscription %s\n"), + sub_name); + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop subscriber " + "subscription %s: %s\n"), + sub_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(drop_res); + } + } + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not list subscriptions on " + "subscriber \"%s\": %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(res); + destroyPQExpBuffer(query); + PQfinish(n3_conn); + + return fully_cleaned; +} + +/* + * Drop resources this run may have created on the source and each peer: + * the source's replication slot and reverse subscription (if recorded as + * created), and each peer's replication slot and reverse subscription. + * Unlike n3 (a freshly-provisioned node with no unrelated subscriptions, + * see cleanup_verified_subscriber_node()), the source and peers have their + * own pre-existing state that must not be touched -- so every drop here is + * gated by a flag or an LSN this run itself recorded, and reverse + * subscriptions are targeted by name via drop_reverse_sub() rather than + * dropping everything found. + */ +static bool +cleanup_upstream_node_resources(BidirectionalState *state, const char *subscriber_name, + const char *source_dsn) +{ + PGconn *source_conn; + PGresult *res; + int i; + bool fully_cleaned = true; + + source_conn = PQconnectdb(source_dsn); + if (PQstatus(source_conn) != CONNECTION_OK) + { + if (state->source_slot_name && state->source_slot_name[0]) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; slot %s " + "may still exist: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } + PQfinish(source_conn); + source_conn = NULL; + } + + /* Drop source replication slot if it was created */ + if (source_conn && state->source_slot_name && state->source_slot_name[0]) + { + if (drop_logical_slot_if_exists(source_conn, state->source_slot_name, + "the source") == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + /* + * Drop the reverse subscription on the source if this run recorded + * having created it. Unlike n3 (whose block drops every subscription + * it finds, since a fresh n3 has no other legitimate ones), the source + * has its own pre-existing, unrelated subscriptions that must not be + * touched -- so this is gated by the flag and targets the specific + * reverse subscription by name. + */ + if (source_conn && state->source_reverse_sub_created) + { + res = debug_exec(source_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + print_msg(VERBOSITY_NORMAL, + _("warning: could not determine the source's node name; " + "its reverse subscription may still exist: %s\n"), + PQerrorMessage(source_conn)); + fully_cleaned = false; + } + else + { + char *source_node_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + if (drop_reverse_sub(source_conn, source_node_name, subscriber_name, + "the source") == CLEANUP_DROP_FAILED) + fully_cleaned = false; + pg_free(source_node_name); + } + } + + /* Per-peer: drop slot and any reverse subscription */ + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + + if (!peer->dsn || !peer->dsn[0]) + continue; + + /* + * Only attempt to drop -- and only require connectivity for -- + * resources this run actually recorded as created. Slot/sub names + * are deterministic, not per-run unique, so --cleanup must not + * touch a same-named resource from an unrelated join, nor report + * "incomplete" over a peer that was never touched. + */ + if ((!peer->slot_creation_lsn || !peer->slot_creation_lsn[0]) && + !peer->reverse_sub_created) + continue; + + peer_conn = PQconnectdb(peer->dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to peer \"%s\"; its slot/" + "subscription may still exist: %s\n"), + peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; + PQfinish(peer_conn); + continue; + } + + if (peer->slot_creation_lsn && peer->slot_creation_lsn[0] && + peer->slot_name && peer->slot_name[0]) + { + if (drop_logical_slot_if_exists(peer_conn, peer->slot_name, + peer->node_name) == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + /* + * Drop the reverse subscription (peer -> new subscriber) only if + * this run recorded having created it. The sub_drop second + * argument is ifexists=true, so an absent subscription is not an + * error -- only an actual query failure counts against + * fully_cleaned. + */ + if (peer->reverse_sub_created) + { + if (drop_reverse_sub(peer_conn, peer->node_name, subscriber_name, + peer->node_name) == CLEANUP_DROP_FAILED) + fully_cleaned = false; + } + + PQfinish(peer_conn); + print_msg(VERBOSITY_NORMAL, + _(" cleaned up peer %s\n"), peer->node_name); + } + + if (source_conn) + PQfinish(source_conn); + + return fully_cleaned; +} + +/* + * Idempotently remove bidirectional join state from all reachable nodes: + * n3 itself, the source, and each peer, via + * cleanup_verified_subscriber_node() and cleanup_upstream_node_resources(). + * Connectivity and drop failures are logged as warnings, not fatal, so + * cleanup attempts every remaining resource -- but each failure is + * tracked, and the function returns true only if every recorded resource + * was confirmed gone. The manifest/sidecar record (the only way to + * retry) is removed only on a true return; an incomplete cleanup keeps + * it and the caller exits non-zero. + */ +static bool +cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir) +{ + bool fully_cleaned = true; + + print_msg(VERBOSITY_NORMAL, + _("Cleaning up partial bidirectional join state ...\n")); + + if (!cleanup_verified_subscriber_node(state, subscriber_name)) + fully_cleaned = false; + + if (!cleanup_upstream_node_resources(state, subscriber_name, source_dsn)) + fully_cleaned = false; + + /* + * Stop n3's postmaster unconditionally (not gated by --force, which + * only governs removing the data directory). check_data_dir() and + * check_reused_data_dir_is_safe() explicitly support resuming a join + * into this same data_dir after a failed attempt, and that resume + * path (main(), the "start -l ..." pg_ctl call before catchup) assumes + * postgres is not already running here; leaving it up after + * `--cleanup` would make the very next retry fail outright. Every + * drop above already ran while n3 was still reachable -- including + * the source's and each peer's reverse subscription, whose provider + * is n3, so dropping them needs a live connection back to it -- so + * this is just shutdown, not a substitute for them. + */ + stop_postgres_in_data_dir(); + + /* + * The data directory a partial run may have created via basebackup. + * Never touch it without --force. + */ + if (!remove_data_dir_if_forced(force_rm_datadir)) + fully_cleaned = false; + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: some resource(s) above could not be " + "confirmed removed. Keeping the manifest/sidecar record " + "so --cleanup can be retried.\n")); + return false; + } + + /* + * Every remote/local resource above was confirmed gone; now remove the + * retry record(s) themselves. An unexpected removal failure here + * (anything but ENOENT, i.e. already gone) must also flip + * fully_cleaned -- otherwise the caller reports success and exits 0 + * while a stale record that still references now-removed resources + * lingers on disk, which a later --cleanup could misread as current. + */ + if (state->manifest_path && state->manifest_path[0]) + { + if (unlink(state->manifest_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove manifest %s: %s\n"), + state->manifest_path, strerror(errno)); + fully_cleaned = false; + } + } + + /* + * Also remove any pending-cleanup sidecar, even if it wasn't the file + * that drove this cleanup: a stale one left behind by an earlier run + * whose own sidecar-unlink failed could otherwise be misread as + * current by a later --cleanup once the manifest above is gone, + * reporting resources as still-pending that were, in fact, already + * confirmed removed here. + */ + if (data_dir != NULL && data_dir[0]) + { + char sidecar_path[MAXPGPATH]; + + snprintf(sidecar_path, MAXPGPATH, "%s.spock_bidir_pending.json", data_dir); + if (unlink(sidecar_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed pending sidecar %s\n"), sidecar_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove pending sidecar %s: %s\n"), + sidecar_path, strerror(errno)); + fully_cleaned = false; + } + } + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: the manifest or sidecar record could " + "not be removed even though every resource it tracked " + "was confirmed gone. Retry --cleanup to remove the " + "stale record.\n")); + return false; + } + + print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); + return true; +} + + +/* + * Phase 1: parse command-line options into ctx, then validate them and + * derive the bidirectional manifest/sidecar paths. Handles --help + * directly (exits). Self-contained -- no phase runs before this one. + */ +static void +parse_options(int argc, char **argv, SubscriberCreateContext *ctx) +{ + int i; + int c; + int optindex; + + static struct option long_options[] = { + {"subscriber-name", required_argument, NULL, 'n'}, + {"pgdata", required_argument, NULL, 'D'}, + {"provider-dsn", required_argument, NULL, 1}, + {"subscriber-dsn", required_argument, NULL, 2}, + {"replication-sets", required_argument, NULL, 3}, + {"postgresql-conf", required_argument, NULL, 4}, + {"hba-conf", required_argument, NULL, 5}, + {"recovery-conf", required_argument, NULL, 6}, + {"stop", no_argument, NULL, 's'}, + {"drop-slot-if-exists", no_argument, NULL, 7}, + {"apply-delay", required_argument, NULL, 8}, + {"databases", required_argument, NULL, 9}, + {"extra-basebackup-args", required_argument, NULL, 10}, + {"text-types", no_argument, NULL, 11}, + {"bidirectional", no_argument, NULL, 12}, + {"stall-timeout", required_argument, NULL, 13}, + {"max-wait", required_argument, NULL, 14}, + {"cleanup", no_argument, NULL, 15}, + {"force", no_argument, NULL, 16}, + {"postgresql-auto-conf", required_argument, NULL, 17}, + {NULL, 0, NULL, 0} + }; + + /* check for --help */ + if (argc > 1) + { + for (i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-?") == 0) + { + usage(); + exit(0); + } + } + } + + /* Option parsing and validation */ + while ((c = getopt_long(argc, argv, "D:n:sv", long_options, &optindex)) != -1) + { + switch (c) + { + case 'D': + data_dir = expand_tilde(pg_strdup(optarg)); + break; + case 'n': + ctx->subscriber_name = pg_strdup(optarg); + break; + case 1: + ctx->base_prov_connstr = pg_strdup(optarg); + break; + case 2: + ctx->base_sub_connstr = pg_strdup(optarg); + break; + case 3: + ctx->replication_sets = validate_replication_set_input(pg_strdup(optarg)); + break; + case 4: + ctx->postgresql_conf = validated_existing_path(optarg, "postgresql.conf"); + break; + case 5: + ctx->pg_hba_conf = validated_existing_path(optarg, "pg_hba.conf"); + break; + case 6: + ctx->recovery_conf = validated_existing_path(optarg, "recovery configuration"); + break; + case 'v': + verbosity++; + break; + case 's': + ctx->stop = true; + break; + case 7: + ctx->drop_slot_if_exists = true; + break; + case 8: + ctx->apply_delay = parse_checked_int(optarg, "apply-delay"); + break; + case 9: + ctx->databases = pg_strdup(optarg); + break; + case 10: + ctx->extra_basebackup_args = pg_strdup(optarg); + validate_extra_basebackup_args(ctx->extra_basebackup_args); + break; + case 11: + ctx->force_text_transfer = true; + break; + case 12: + ctx->bidir.enabled = true; + break; + case 13: + ctx->bidir.stall_timeout = parse_checked_int(optarg, "stall-timeout"); + if (ctx->bidir.stall_timeout <= 0) + die(_("--stall-timeout must be a positive integer")); + break; + case 14: + ctx->bidir.max_wait = parse_checked_int(optarg, "max-wait"); + if (ctx->bidir.max_wait < 0) + die(_("--max-wait must be a non-negative integer")); + break; + case 15: + ctx->bidir.cleanup_mode = true; + break; + case 16: + ctx->bidir.force_cleanup = true; + break; + case 17: + ctx->postgresql_auto_conf = validated_existing_path(optarg, "postgresql.auto.conf"); + break; + default: + fprintf(stderr, _("Unknown option\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* + * Sanity checks + */ + + if (data_dir == NULL) + { + fprintf(stderr, _("No data directory specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + else if (ctx->subscriber_name == NULL && !ctx->bidir.cleanup_mode) + { + fprintf(stderr, _("No subscriber name specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + if (ctx->bidir.cleanup_mode && !ctx->bidir.enabled) + die(_("--cleanup requires --bidirectional.\n")); + + if (ctx->bidir.force_cleanup && !ctx->bidir.cleanup_mode) + die(_("--force requires --cleanup.\n")); + + if (!ctx->bidir.cleanup_mode && (!ctx->base_prov_connstr || !strlen(ctx->base_prov_connstr))) + die(_("Provider connection string must be specified.\n")); + if (!ctx->bidir.cleanup_mode && + (!ctx->base_sub_connstr || !strlen(ctx->base_sub_connstr))) + die(_("Subscriber connection string must be specified: --subscriber-dsn " + "is used both for the tool's own connection to the newly " + "created node and, with --bidirectional, as the externally-" + "reachable address registered via spock.node_create() for " + "peers to connect back to it.\n")); + + if (ctx->apply_delay < 0) + die(_("Apply delay cannot be negative.\n")); + + if (ctx->apply_delay > MAX_APPLY_DELAY) + die(_("Apply delay cannot be more than %d.\n"), MAX_APPLY_DELAY); + + if (ctx->bidir.enabled) + { + /* + * n3 is joining an existing mesh, so its subscriptions must select + * exactly what the mesh already replicates; replication_sets is + * derived from the source's own subscriptions below instead. + */ + if (ctx->replication_sets != NULL) + die(_("--replication-sets cannot be combined with --bidirectional; " + "the joining node's replication sets are detected " + "automatically from the cluster it is joining.\n")); + } + else if (!ctx->replication_sets || !strlen(ctx->replication_sets)) + ctx->replication_sets = "default,default_insert_only,ddl_sql"; + + /* Build the manifest path from --pgdata */ + if (ctx->bidir.enabled || ctx->bidir.cleanup_mode) + { + snprintf(ctx->bidir_manifest_path, MAXPGPATH, + "%s/spock_bidirectional_manifest.json", data_dir); + ctx->bidir.manifest_path = ctx->bidir_manifest_path; + /* + * Sidecar path for the source slot orphan-protection record (see + * the write near source-slot creation below) -- lives next to, not + * inside, data_dir, since data_dir must still be empty when this is + * first written (pg_basebackup requires an empty target directory). + */ + snprintf(ctx->bidir_pending_path, MAXPGPATH, + "%s.spock_bidir_pending.json", data_dir); + if (ctx->bidir.stall_timeout == 0) + ctx->bidir.stall_timeout = 600; + } +} + +/* + * Phase 2: if --cleanup was requested, read the manifest (or, failing + * that, the pending-cleanup sidecar), remove partial join state, and + * exit -- this phase never returns when ctx->bidir.cleanup_mode is set. + * A no-op otherwise. + */ +static void +run_cleanup_mode_if_requested(SubscriberCreateContext *ctx) +{ + char *sub_name = NULL; + char *db = NULL; + char *src_dsn = NULL; + + if (!ctx->bidir.cleanup_mode) + return; + + if (read_manifest(ctx->bidir.manifest_path, &ctx->bidir, &sub_name, &db, &src_dsn)) + exit(cleanup_partial_state(&ctx->bidir, sub_name, db, src_dsn, + ctx->bidir.force_cleanup) ? 0 : 1); + + /* + * No full manifest -- basebackup may never have completed. Fall + * back to the pending-cleanup sidecar written right after source + * slot creation, so a slot orphaned by a failed/interrupted backup + * is still reachable by --cleanup. + */ + if (read_manifest(ctx->bidir_pending_path, &ctx->bidir, &sub_name, &db, &src_dsn)) + /* cleanup_partial_state() removes the sidecar itself on success. */ + exit(cleanup_partial_state(&ctx->bidir, sub_name, db, src_dsn, + ctx->bidir.force_cleanup) ? 0 : 1); + + /* + * Neither record exists -- there's no slot/subscription bookkeeping + * to act on, e.g. because the run died before the pending sidecar + * was even written. But an orphaned data_dir can still be sitting + * there from that attempt, and --force is an explicit instruction + * to remove it: don't leave it behind just because there was + * nothing to read. + */ + if (ctx->bidir.force_cleanup && data_dir != NULL && data_dir[0] && + file_exists(data_dir)) + { + fprintf(stderr, + _("No manifest found at %s or %s; no slot/subscription " + "state to clean up, but --force was given -- removing " + "data directory %s.\n"), + ctx->bidir.manifest_path, ctx->bidir_pending_path, data_dir); + exit(remove_data_dir_if_forced(true) ? 0 : 1); + } + + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), + ctx->bidir.manifest_path, ctx->bidir_pending_path); + exit(0); +} + +/* + * Phase 3: resolve --databases/--provider-dsn into ctx->database_list, + * enforcing --bidirectional's single-database restriction. + */ +static void +resolve_database_targets(SubscriberCreateContext *ctx) +{ + /* Init random numbers used for slot suffixes, etc */ + srand(time(NULL)); + + /* Parse database list or connection string. */ + if (ctx->databases != NULL) + { + ctx->database_list = get_database_list(ctx->databases, &ctx->n_databases); + } + else + { + char *dbname = get_connstr_dbname(ctx->base_prov_connstr); + + if (!dbname) + die(_("Either provider connection string must contain database " + "name or --databases option must be specified.\n")); + + ctx->n_databases = 1; + ctx->database_list = palloc(ctx->n_databases * sizeof(char *)); + ctx->database_list[0] = dbname; + } + + /* + * Single database only: all join state is per-database, and the + * physical-backup/recovery path operates on one data directory. + * Reject a multi-database request rather than silently joining only + * database_list[0]. Separate from check_single_spock_database() + * below, which checks the instance for spock on other databases. + */ + if (ctx->bidir.enabled && ctx->n_databases > 1) + die(_("--bidirectional supports a single database only; " + "%d were named via --databases/--provider-dsn.\n"), + ctx->n_databases); + + ctx->slot_names = palloc(ctx->n_databases * sizeof(char *)); +} + +/* + * Phase 4: validate every resolved database's connection strings before + * doing anything expensive. ctx->prov_connstr/ctx->sub_connstr here are + * just scratch -- the values used by later phases are recomputed there. + */ +static void +validate_connection_strings(SubscriberCreateContext *ctx) +{ + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + if (!ctx->prov_connstr || !strlen(ctx->prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + if (!ctx->bidir.enabled) + { + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + if (!ctx->sub_connstr || !strlen(ctx->sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } + } +} + +/* + * Phase 5: create the postgres startup log file, then, for each + * resolved database, connect to the provider and create a replication + * slot. --bidirectional additionally discovers peers, checks join + * preconditions, and persists a pending-cleanup sidecar before the base + * backup even starts -- then breaks after database_list[0], since + * --bidirectional is single-database only (enforced in + * resolve_database_targets()). Sets ctx->remote_info (the last/only + * database processed) and ctx->slot_names. + */ +static void +create_replication_slots(SubscriberCreateContext *ctx) +{ + int dbnum; + int logfd; + + /* + * Create log file where new postgres instance will log to while being + * initialized. + */ + logfd = open("spock_create_subscriber_postgres.log", O_CREAT | O_RDWR, + S_IRUSR | S_IWUSR); + if (logfd == -1) + { + die(_("Creating spock_create_subscriber_postgres.log failed: %s"), + strerror(errno)); + } + /* Safe to close() unchecked, we didn't write */ + (void) close(logfd); + + /* Let's start the real work... */ + print_msg(VERBOSITY_NORMAL, _("%s: starting ...\n"), progname); + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + if (!ctx->prov_connstr || !strlen(ctx->prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + /* Read the remote server indetification. */ + print_msg(VERBOSITY_NORMAL, + _("Getting information for database %s ...\n"), db); + provider_conn = connectdb(ctx->prov_connstr); + ctx->remote_info = get_remote_info(provider_conn); + + /* + * --bidirectional: discover peers, verify preconditions, then + * continue into the physical-backup pipeline below using the + * "sub__" slot naming convention. Manifest + * write is deferred until after the basebackup; see the comment + * there. + */ + if (ctx->bidir.enabled) + { + char *source_sub_name; + + /* + * Inherit the replication sets already in use by the cluster + * being joined, rather than accept a separately specified + * list -- see the die() in parse_options() that rejects + * --replication-sets together with --bidirectional. + */ + ctx->replication_sets = get_source_mesh_replication_sets(provider_conn); + print_msg(VERBOSITY_VERBOSE, + _("Replication sets inherited from the existing " + "cluster: %s\n"), ctx->replication_sets); + + ctx->bidir.num_peers = discover_peer_nodes(provider_conn, + ctx->remote_info->node_name, + ctx->subscriber_name, db, + &ctx->bidir.peers); + { + int pi; + + for (pi = 0; pi < ctx->bidir.num_peers; pi++) + print_msg(VERBOSITY_DEBUG, + _("Discovered peer \"%s\" (dsn \"%s\", slot \"%s\")\n"), + ctx->bidir.peers[pi].node_name, ctx->bidir.peers[pi].dsn, + ctx->bidir.peers[pi].slot_name); + } + check_preconditions(provider_conn, ctx->remote_info->node_name, + ctx->bidir.peers, ctx->bidir.num_peers); + check_single_spock_database(provider_conn, ctx->base_prov_connstr, db); + check_no_native_subscriptions(provider_conn); + ctx->use_existing_data_dir = check_data_dir(data_dir, ctx->remote_info); + if (ctx->use_existing_data_dir) + check_reused_data_dir_is_safe(data_dir, ctx->remote_info); + + source_sub_name = sub_name_for(ctx->subscriber_name, ctx->remote_info->node_name); + + print_msg(VERBOSITY_NORMAL, + _("Creating source replication slot in database %s ...\n"), db); + print_msg(VERBOSITY_DEBUG, + _("Creating replication slot on source \"%s\" for future " + "subscription \"%s\"\n"), ctx->remote_info->node_name, source_sub_name); + ctx->bidir.source_slot_name = initialize_replication_slot(provider_conn, + ctx->remote_info->dbname, + ctx->remote_info->node_name, + source_sub_name, + ctx->drop_slot_if_exists); + print_msg(VERBOSITY_DEBUG, _("Source replication slot created: \"%s\"\n"), + ctx->bidir.source_slot_name); + pg_free(source_sub_name); + + /* + * Persist a pending-cleanup record now, before the base backup + * even starts: the source slot above already exists on the + * remote node, and a failed/interrupted backup would otherwise + * orphan it with nothing for --cleanup to find (the real + * manifest can't be written yet -- data_dir must stay empty for + * pg_basebackup). Superseded and removed once the real + * manifest is written below. + */ + ctx->bidir.manifest_path = ctx->bidir_pending_path; + write_manifest(&ctx->bidir, ctx->subscriber_name, db, ctx->base_prov_connstr); + ctx->bidir.manifest_path = ctx->bidir_manifest_path; + + PQfinish(provider_conn); + provider_conn = NULL; + break; /* single-database only, enforced above */ + } + + /* only need to do this piece once */ + + if (dbnum == 0) + { + ctx->use_existing_data_dir = check_data_dir(data_dir, ctx->remote_info); + + if (ctx->use_existing_data_dir) + check_reused_data_dir_is_safe(data_dir, ctx->remote_info); + } + + /* + * Create replication slots on remote node. + */ + print_msg(VERBOSITY_NORMAL, + _("Creating replication slot in database %s ...\n"), db); + ctx->slot_names[dbnum] = initialize_replication_slot(provider_conn, + ctx->remote_info->dbname, + ctx->remote_info->node_name, + ctx->subscriber_name, + ctx->drop_slot_if_exists); + PQfinish(provider_conn); + provider_conn = NULL; + } +} + +/* + * Phase 6: take (or reuse) the physical base backup into data_dir, then + * write the real manifest -- deferred until here because pg_basebackup + * requires an empty target directory, and a manifest file in data_dir + * earlier would make it look non-empty. The pending-cleanup sidecar + * written in create_replication_slots() covers the gap between then and + * now; it's superseded and removed here. + */ +static void +run_basebackup_and_write_manifest(SubscriberCreateContext *ctx) +{ + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, ctx->database_list[0]); + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, ctx->database_list[0]); + + if (!ctx->use_existing_data_dir) + print_msg(VERBOSITY_DEBUG, + _("Taking a physical base backup from \"%s\" into \"%s\"\n"), + ctx->prov_connstr, data_dir); + else + print_msg(VERBOSITY_DEBUG, + _("Reusing existing data directory \"%s\" (already a basebackup " + "of this source)\n"), data_dir); + initialize_data_dir(data_dir, + ctx->use_existing_data_dir ? NULL : ctx->prov_connstr, + ctx->postgresql_conf, ctx->postgresql_auto_conf, ctx->pg_hba_conf, + ctx->extra_basebackup_args); + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + + if (ctx->bidir.enabled) + { + write_manifest(&ctx->bidir, ctx->subscriber_name, ctx->database_list[0], ctx->base_prov_connstr); + if (unlink(ctx->bidir_pending_path) != 0 && errno != ENOENT) + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove superseded pending sidecar " + "%s: %s\n"), ctx->bidir_pending_path, strerror(errno)); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "source slot created, manifest written to %s.\n"), + ctx->bidir.num_peers, ctx->bidir.manifest_path); + } +} + +/* + * Phase 7: create a restore point on the provider, bring the subscriber + * up to it via physical recovery, and wait for it to start accepting + * connections -- proof it has caught up. Sets ctx->remote_lsn. + */ +static void +catchup_to_restore_point(SubscriberCreateContext *ctx) +{ + PQExpBuffer recoveryconfcontents = createPQExpBuffer(); + char *restore_point_name; + int pg_ctl_ret; + + restore_point_name = generate_restore_point_name(); + + print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), + restore_point_name); + provider_conn = connectdb(ctx->prov_connstr); + ctx->remote_lsn = create_restore_point(provider_conn, restore_point_name); + PQfinish(provider_conn); + provider_conn = NULL; + + /* + * Get subscriber db to consistent state (for lsn after slot creation). + */ + print_msg(VERBOSITY_NORMAL, + _("Bringing subscriber node to the restore point ...\n")); + if (ctx->recovery_conf) + { + CopyConfFile(ctx->recovery_conf, "postgresql.auto.conf", true); + } + else + { + appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", + escape_single_quotes_ascii(ctx->prov_connstr)); + } + appendPQExpBuffer(recoveryconfcontents, "recovery_target_name = '%s'\n", restore_point_name); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_inclusive = true\n"); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_action = promote\n"); + WriteRecoveryConf(recoveryconfcontents); + + free(restore_point_name); + restore_point_name = NULL; + + /* + * Start subscriber node with spock disabled, and wait until it starts + * accepting connections which means it has caught up to the restore point. + * + * TODO: for --bidirectional this node should be network-quarantined + * (private socket/listen address, or a restrictive pg_hba.conf) from + * this first startup through the end of the join -- spock.readonly = + * 'local' (set later) blocks writes but not reads or peer probes. Not + * implemented: --subscriber-dsn must be directly reachable, and the + * tool's own connections use that same DSN throughout, so restricting + * listen_addresses here would also lock the tool itself out. + */ + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + + wait_primary_connection(ctx->sub_connstr, + ctx->bidir.enabled ? ctx->bidir.stall_timeout : 0, + ctx->bidir.enabled ? ctx->bidir.max_wait : 0); +} + +/* + * Phase 8: strip the spock configuration pg_basebackup copied over from + * the provider -- for --bidirectional, this also gives n3 its own + * system identifier, verifies --subscriber-dsn actually reaches it, and + * captures repset/table/sequence state before the catalog strip (sets + * ctx->bidir.node_sysid, ctx->capture); for the plain path, just clears + * each database's spock configuration. Ends by stopping postgres so it + * can be restarted with spock loaded. + */ +static void +strip_subscriber_catalog(SubscriberCreateContext *ctx) +{ + int pg_ctl_ret; + + /* + * Clean any per-node data that were copied by pg_basebackup. + */ + print_msg(VERBOSITY_VERBOSE, + _("Removing old spock configuration ...\n")); + + if (ctx->bidir.enabled) + { + Oid source_nodeid; + char *expected_sysid; + + /* + * Give n3 its own permanent identity now, right after promotion + * and before any catalog mutation: a physical backup preserves + * the source's system identifier, which risks stray WAL from one + * cluster being mistaken for the other's, and until reset makes + * system_identifier useless for proving a connection actually + * reaches n3 rather than the source. + */ + print_msg(VERBOSITY_NORMAL, + _("Assigning a new system identifier to the subscriber node...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop before resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + + { + sigset_t block_set, + old_set; + + /* + * Neither step below is safe to interrupt -- both write + * pg_control/WAL directly, and signal_handler() -> die() is + * not async-signal-safe. A signal landing mid-write could + * corrupt pg_control with no repair short of --cleanup + * --force. Block both signals across this pair of calls; + * any that arrives is deferred until right after. + */ + sigemptyset(&block_set); + sigaddset(&block_set, SIGINT); + sigaddset(&block_set, SIGTERM); + sigprocmask(SIG_BLOCK, &block_set, &old_set); + + expected_sysid = reset_subscriber_sysid(data_dir); + run_pg_resetwal(data_dir); + + sigprocmask(SIG_SETMASK, &old_set, NULL); + } + + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup after resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_connection(ctx->sub_connstr); + + subscriber_conn = connectdb(ctx->sub_connstr); + + /* + * --subscriber-dsn is expected to point directly at this node; + * verify that cheaply before running anything destructive, rather + * than trusting it silently. Now that n3 has just been given its + * own system identifier above, a straightforward comparison is a + * valid proof the connection reaches n3 and not the source or any + * other server -- unlike before the reset, nothing else could + * share it. + */ + { + PGresult *sysid_res = debug_exec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + bool mismatch; + + if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) + { + PQclear(sysid_res); + die(_("could not verify --subscriber-dsn connects to this node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + mismatch = strcmp(PQgetvalue(sysid_res, 0, 0), expected_sysid) != 0; + PQclear(sysid_res); + if (mismatch) + die(_("--subscriber-dsn does not connect to the node at \"%s\": " + "system identifier mismatch. This can happen if the DSN " + "routes to the source node or another server; refusing " + "to run catalog operations against it.\n"), data_dir); + } + + /* + * Persist n3's own system identifier so --cleanup can re-verify + * node_dsn still reaches this same node later, rather than trusting + * a possibly stale manifest to still point at the right server. + */ + ctx->bidir.node_sysid = expected_sysid; + + /* Capture repset/table/sequence state before the catalog strip. */ + source_nodeid = get_local_node_id(subscriber_conn); + print_msg(VERBOSITY_DEBUG, + _("Capturing replication-set/table/sequence membership for local " + "node id %u before dropping the spock extension\n"), source_nodeid); + capture_catalog_state(subscriber_conn, source_nodeid, &ctx->capture); + print_msg(VERBOSITY_DEBUG, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s)\n"), + ctx->capture.num_repsets, ctx->capture.num_tables, ctx->capture.num_sequences); + + /* Drop all origins, then guarded DROP EXTENSION. */ + print_msg(VERBOSITY_DEBUG, + _("Dropping replication origins and the spock extension (checking " + "pg_depend first for non-spock objects CASCADE would collaterally " + "drop)\n")); + remove_unwanted_data_bidir(subscriber_conn, &ctx->capture); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + else + { + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + + if (!ctx->sub_connstr || !strlen(ctx->sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(ctx->sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } + + /* Stop Postgres so we can start it again with spock (shared_preload_libraries) loaded. */ + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + wait_postmaster_shutdown(); +} + +/* + * Phase 9: restart postgres with spock loaded, then bring the node + * live -- for --bidirectional, the full go-live pipeline (local node + * creation, readonly, replication-set restore, catchup subscription, + * coverage barrier, forwarding clear, direct/reverse subscriptions, + * dataflow verification, readonly lift); for the plain path, the + * ordinary extension/origin/subscription creation per database. + */ +static void +restart_with_spock_and_activate(SubscriberCreateContext *ctx) +{ + int pg_ctl_ret; + + /* + * Start the node again, now with spock active so that we can start the + * logical replication. This is final start, so don't log to to special log + * file anymore. + */ + print_msg(VERBOSITY_NORMAL, + _("Initializing spock on the subscriber node:\n")); + + pg_ctl_ret = run_pg_ctl("start"); + if (pg_ctl_ret != 0) + die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); + wait_postmaster_connection(ctx->bidir.enabled ? ctx->sub_connstr : ctx->base_sub_connstr); + + if (ctx->bidir.enabled) + { + char *db = ctx->database_list[0]; + + subscriber_conn = connectdb(ctx->sub_connstr); + + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the local node, then immediately go read-only -- no + * window where n3 is reachable/writable before that lands. No + * origin creation here; the catchup subscription creates it + * later. + * + * dsn is --subscriber-dsn (sub_connstr) -- the externally-reachable + * address other nodes use to connect back, not a separate + * --node-dsn option. + */ + print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), + ctx->subscriber_name); + print_msg(VERBOSITY_DEBUG, _("Registering node \"%s\" with dsn \"%s\"\n"), + ctx->subscriber_name, ctx->sub_connstr); + { + PQExpBuffer nodequery = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(nodequery, + "SELECT spock.node_create(node_name := %s, dsn := %s)", + PQescapeLiteral(subscriber_conn, ctx->subscriber_name, + strlen(ctx->subscriber_name)), + PQescapeLiteral(subscriber_conn, ctx->sub_connstr, + strlen(ctx->sub_connstr))); + res = debug_exec(subscriber_conn, nodequery->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create local node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + destroyPQExpBuffer(nodequery); + } + + print_msg(VERBOSITY_NORMAL, _("Setting spock.readonly = 'local'...\n")); + set_spock_readonly(subscriber_conn, "local"); + + /* Restore what was captured before the catalog strip. */ + print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + print_msg(VERBOSITY_DEBUG, + _("Restoring %d replication set(s), %d table membership(s), " + "%d sequence(s) onto node \"%s\"\n"), + ctx->capture.num_repsets, ctx->capture.num_tables, ctx->capture.num_sequences, + ctx->subscriber_name); + restore_replication_sets(subscriber_conn, &ctx->capture); + + ctx->bidir.source_restore_lsn = pg_strdup(ctx->remote_lsn); + ctx->bidir.node_dsn = ctx->sub_connstr; + write_manifest(&ctx->bidir, ctx->subscriber_name, db, ctx->base_prov_connstr); + + { + char *source_sub_name; + char *target_lsn; + + source_sub_name = sub_name_for(ctx->subscriber_name, ctx->remote_info->node_name); + + print_msg(VERBOSITY_NORMAL, _("Creating catchup subscription to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Creating subscription \"%s\" to source \"%s\" using slot " + "\"%s\", forward_origins={all}, enabled=false\n"), + source_sub_name, ctx->prov_connstr, ctx->bidir.source_slot_name); + create_catchup_subscription(subscriber_conn, source_sub_name, ctx->prov_connstr, + ctx->replication_sets, ctx->bidir.source_slot_name, + ctx->bidir.source_restore_lsn); + print_msg(VERBOSITY_DEBUG, + _("Subscription \"%s\" created, origin advanced to %s, and " + "enabled\n"), source_sub_name, ctx->bidir.source_restore_lsn); + + print_msg(VERBOSITY_NORMAL, _("Creating disabled peer subscriptions...\n")); + create_disabled_peer_subscriptions(subscriber_conn, ctx->bidir.peers, + ctx->bidir.num_peers, ctx->replication_sets); + + print_msg(VERBOSITY_NORMAL, _("Getting catchup target from the source...\n")); + target_lsn = get_catchup_target_lsn(ctx->prov_connstr); + print_msg(VERBOSITY_DEBUG, _("Catchup target LSN: %s\n"), target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Waiting for catchup to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Waiting for subscription \"%s\" (origin \"%s\") to reach " + "LSN %s\n"), source_sub_name, ctx->bidir.source_slot_name, target_lsn); + wait_for_catchup(subscriber_conn, source_sub_name, ctx->bidir.source_slot_name, + target_lsn, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + pg_free(target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Establishing peer coverage barrier...\n")); + establish_peer_coverage_barrier(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, source_sub_name, + ctx->bidir.source_slot_name, ctx->subscriber_name, db, + ctx->base_prov_connstr, ctx->bidir.stall_timeout, + ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Clearing forwarding on the catchup subscription...\n")); + clear_forwarding(subscriber_conn, ctx->prov_connstr, source_sub_name, + ctx->bidir.source_slot_name, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Enabling direct peer subscriptions...\n")); + enable_peer_subs(subscriber_conn, ctx->bidir.peers, ctx->bidir.num_peers, + ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Creating reverse subscriptions...\n")); + create_reverse_subscriptions(&ctx->bidir, ctx->subscriber_name, ctx->sub_connstr, + ctx->replication_sets, ctx->prov_connstr, + ctx->remote_info->node_name, db, ctx->base_prov_connstr); + + print_msg(VERBOSITY_NORMAL, _("Waiting for reverse subscriptions to be ready...\n")); + wait_for_reverse_subs_ready(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, ctx->subscriber_name, + ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Verifying bidirectional replication...\n")); + verify_bidirectional_dataflow(&ctx->bidir, subscriber_conn, ctx->prov_connstr, + ctx->remote_info->node_name, source_sub_name, + ctx->subscriber_name, ctx->bidir.stall_timeout, ctx->bidir.max_wait); + + pg_free(source_sub_name); + + print_msg(VERBOSITY_NORMAL, _("Lifting read-only mode...\n")); + set_spock_readonly(subscriber_conn, "off"); + } + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + + print_msg(VERBOSITY_NORMAL, + _("Bidirectional join complete: node \"%s\" is a live, verified " + "bidirectional member of the cluster.\n"), + ctx->subscriber_name); + } + else + { + int dbnum; + + for (dbnum = 0; dbnum < ctx->n_databases; dbnum++) + { + char *db = ctx->database_list[dbnum]; + + ctx->sub_connstr = get_connstr(ctx->base_sub_connstr, db); + ctx->prov_connstr = get_connstr(ctx->base_prov_connstr, db); + + subscriber_conn = connectdb(ctx->sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which we + * already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, ctx->slot_names[dbnum], ctx->remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + ctx->subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), ctx->replication_sets); + + spock_subscribe(subscriber_conn, ctx->subscriber_name, ctx->sub_connstr, + ctx->prov_connstr, ctx->replication_sets, ctx->apply_delay, + ctx->force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } +} + +int +main(int argc, char **argv) +{ + SubscriberCreateContext ctx = {0}; + + argv0 = argv[0]; + progname = get_progname(argv[0]); + pg_logging_init(argv[0]); + start_time = time(NULL); + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + + parse_options(argc, argv, &ctx); + run_cleanup_mode_if_requested(&ctx); /* exits if --cleanup was given */ + + resolve_database_targets(&ctx); + validate_connection_strings(&ctx); + create_replication_slots(&ctx); + run_basebackup_and_write_manifest(&ctx); + catchup_to_restore_point(&ctx); + strip_subscriber_catalog(&ctx); + restart_with_spock_and_activate(&ctx); + + /* If user does not want the node to be running at the end, stop it. */ + if (ctx.stop) + { + int pg_ctl_ret; + + print_msg(VERBOSITY_NORMAL, _("Stopping the subscriber node ...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Stopping postgres after successful subscribtion failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, _("All done\n")); + + return 0; +} + + +/* + * Print help. + */ +static void +usage(void) +{ + printf(_("%s create new spock subscriber from basebackup of provider.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]...\n"), progname); + printf(_("\nGeneral options:\n")); + printf(_(" -D, --pgdata=DIRECTORY data directory to be used for new node,\n")); + printf(_(" can be either empty/non-existing directory,\n")); + printf(_(" or directory populated using\n")); + printf(_(" pg_basebackup -X stream command\n")); + printf(_(" --databases optional list of databases to replicate\n")); + printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber;\n")); + printf(_(" with --bidirectional, also the externally-\n")); + printf(_(" reachable address peers use to connect back\n")); + printf(_(" to this node once joined (required)\n")); + printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); + printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); + printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); + printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); + printf(_(" -s, --stop stop the server once the initialization is done\n")); + printf(_(" -v increase logging verbosity; repeatable --\n")); + printf(_(" -v -v also traces every query this tool\n")); + printf(_(" runs, with its result status\n")); + printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); + printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_(" --text-types transfer column values as text rather than binary\n")); + printf(_(" (use when provider and subscriber differ in type\n")); + printf(_(" representation or endianness)\n")); + printf(_("\nConfiguration files override:\n")); + printf(_(" --hba-conf path to the new pg_hba.conf\n")); + printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --postgresql-auto-conf settings to override in postgresql.auto.conf\n")); + printf(_(" --recovery-conf path to the template recovery configuration\n")); + printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); + printf(_(" --bidirectional enable bidirectional join plumbing\n")); + printf(_(" --stall-timeout=SECS once PostgreSQL accepts connections, seconds of no\n")); + printf(_(" replay progress before giving up (default 600); does\n")); + printf(_(" not bound PostgreSQL's own startup\n")); + printf(_(" --max-wait=SECS hard ceiling on post-connection catchup wait, seconds\n")); + printf(_(" (default: unbounded); does not bound PostgreSQL's own\n")); + printf(_(" startup\n")); + printf(_(" --cleanup idempotently remove partial join state and exit;\n")); + printf(_(" stops postgres if it is running in --pgdata\n")); + printf(_(" --force with --cleanup, also remove the data directory\n")); + printf(_("\nDuring the join, this node must be network-quarantined (private address /\n")); + printf(_("restrictive pg_hba.conf) by the operator -- via --hba-conf/--postgresql-conf --\n")); + printf(_("until the join completes; the tool does not manage this for you.\n")); +} + +/* + * Print error and exit. + */ +static void +die(const char *fmt,...) +{ + va_list argptr; + va_start(argptr, fmt); + vfprintf(stderr, fmt, argptr); + va_end(argptr); + + if (subscriber_conn) + PQfinish(subscriber_conn); + if (provider_conn) + PQfinish(provider_conn); + + if (get_pgpid()) + { + if (!run_pg_ctl("stop -s")) + { + fprintf(stderr, _("WARNING: postgres seems to be running, but could not be stopped\n")); + } + } + + exit(1); +} + +/* + * Print message to stdout and flush + */ +static void +print_msg(VerbosityLevelEnum level, const char *fmt,...) +{ + if (verbosity >= level) + { + va_list argptr; + va_start(argptr, fmt); + vfprintf(stdout, fmt, argptr); + va_end(argptr); + fflush(stdout); + } +} + +/* + * PQexec() wrapper that logs the query text at VERBOSITY_DEBUG (-v -v) + * before running it, and the resulting status/row count after -- a + * drop-in replacement so every query this tool issues is traceable + * without a separate print_msg() call at each site. Callers still do + * their own PQresultStatus()/die() handling on the result exactly as + * with a plain PQexec() call. + */ +static PGresult * +debug_exec(PGconn *conn, const char *query) +{ + PGresult *res; + + print_msg(VERBOSITY_DEBUG, _(" > %s\n"), query); + res = PQexec(conn, query); + if (verbosity >= VERBOSITY_DEBUG) + { + if (PQresultStatus(res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_DEBUG, _(" < %s (%d row(s))\n"), + PQresStatus(PQresultStatus(res)), PQntuples(res)); + else + print_msg(VERBOSITY_DEBUG, _(" < %s\n"), + PQresStatus(PQresultStatus(res))); + } + + return res; +} + + +/* + * Start pg_ctl with given argument(s) - used to start/stop postgres + * + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * signal this call will die and not return. + */ +static int +run_pg_ctl(const char *arg) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); + + appendPQExpBuffer(cmd, "\"%s\" %s -D \"%s\"", exec_path, arg, data_dir); + + /* Run pg_ctl in silent mode unless we run in debug mode. */ + if (verbosity < VERBOSITY_DEBUG) + appendPQExpBuffer(cmd, " -s"); + + print_msg(VERBOSITY_DEBUG, _("Running pg_ctl: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret)) + return WEXITSTATUS(ret); + else if (WIFSIGNALED(ret)) + die(_("pg_ctl exited with signal %d"), WTERMSIG(ret)); + else + die(_("pg_ctl exited for an unknown reason (system() returned %d)"), ret); + + return -1; +} + + +/* + * Reject --extra-basebackup-args values containing shell control + * characters. The args are appended to a system() command string, so + * semicolons, pipes, backticks, and similar metacharacters would allow + * arbitrary command injection. + */ +static void +validate_extra_basebackup_args(const char *args) +{ + const char *p; + + for (p = args; *p; p++) + { + if (*p == ';' || *p == '|' || *p == '&' || *p == '`' || + *p == '$' || *p == '(' || *p == ')' || + *p == '<' || *p == '>' || *p == '{' || *p == '}' || + *p == '\n' || *p == '\r') + die(_("--extra-basebackup-args contains unsafe shell characters\n")); + } +} + +/* + * Parse optarg as a base-10 integer, dying with a message naming + * opt_name if any part of it isn't numeric or the value doesn't fit in + * an int -- shared by every --