From 86ef5e7d76686a8091e4928f7b8c5a49f2acfa1d Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 01/14] =?UTF-8?q?Add=20spock=5Fcreate=5Fsubscriber=20?= =?UTF-8?q?=E2=80=94=20bootstrap=20a=20subscriber=20from=20a=20physical=20?= =?UTF-8?q?basebackup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a standalone frontend utility that: - Takes a 'pg_basebackup' from the provider and replays to a named restore point. - Creates per-database logical slots on the provider (optionally '--drop-slot-if-exists'). - Starts the new node to catch up with the provider to the restore point. - Installs the Spock extension, creates/advances the replication origin to the restore-point LSN, and creates the subscription to subscribe to the provider. --- .gitignore | 1 - Makefile | 1 + utils/spock_create_subscriber/Makefile | 15 + .../spock_create_subscriber.c | 1775 +++++++++++++++++ 4 files changed, 1791 insertions(+), 1 deletion(-) create mode 100644 utils/spock_create_subscriber/Makefile create mode 100644 utils/spock_create_subscriber/spock_create_subscriber.c diff --git a/.gitignore b/.gitignore index b792ef5ad..cc0c7c0a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ results regression_output tmp_check -spock_create_subscriber .vimrc *.o *.so diff --git a/Makefile b/Makefile index 1105a4fb4..a4c63dbe6 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ EXTENSION = spock PGFILEDESC = "spock - multi-master replication" MODULES = spock_output +SUBDIRS = utils/spock_create_subscriber # Lookup source directory vpath % src src/compat/$(PGVER) diff --git a/utils/spock_create_subscriber/Makefile b/utils/spock_create_subscriber/Makefile new file mode 100644 index 000000000..54e394462 --- /dev/null +++ b/utils/spock_create_subscriber/Makefile @@ -0,0 +1,15 @@ +# Makefile for spock_create_subscriber utility +PG_CONFIG ?= pg_config +PROGRAM = spock_create_subscriber + +PG_CPPFLAGS = -I../../include -I$(shell $(PG_CONFIG) --includedir) +PG_LDFLAGS = -lpq -L$(shell $(PG_CONFIG) --libdir) + +# create symlink to spock_fe.c here +spock_fe.c: ../../src/spock_fe.c + ln -sf $< $@ +OBJS = spock_create_subscriber.o spock_fe.o + +# PGXS +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c new file mode 100644 index 000000000..8147c68b0 --- /dev/null +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -0,0 +1,1775 @@ +/* ------------------------------------------------------------------------- + * + * spock_create_subscriber.c + * Initialize a new spock subscriber from a physical base backup + * + * Copyright (c) 2022-2024, pgEdge, Inc. + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, The Regents of the University of California + * + * ------------------------------------------------------------------------- + */ + +/* dirent.h on port/win32_msvc expects MAX_PATH to be defined */ +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Note the order is important for debian here. */ +#if !defined(pg_attribute_printf) + +/* GCC and XLC support format attributes */ +#if defined(__GNUC__) || defined(__IBMC__) +#define pg_attribute_format_arg(a) __attribute__((format_arg(a))) +#define pg_attribute_printf(f,a) __attribute__((format(PG_PRINTF_ATTRIBUTE, f, a))) +#else +#define pg_attribute_format_arg(a) +#define pg_attribute_printf(f,a) +#endif + +#endif + +#include "libpq-fe.h" +#include "postgres_fe.h" +#include "pqexpbuffer.h" + +#include "getopt_long.h" + +#include "miscadmin.h" + +#include "access/timeline.h" +#include "access/xlog_internal.h" +#include "catalog/pg_control.h" + +#include "spock_fe.h" + +#define MAX_APPLY_DELAY 86400 + +typedef struct RemoteInfo { + Oid nodeid; + char *node_name; + char *sysid; + char *dbname; + char *replication_sets; +} RemoteInfo; + +typedef enum { + VERBOSITY_NORMAL, + VERBOSITY_VERBOSE, + VERBOSITY_DEBUG +} VerbosityLevelEnum; + +static char *argv0 = NULL; +static const char *progname; +static char *data_dir = NULL; +static char pid_file[MAXPGPATH]; +static time_t start_time; +static VerbosityLevelEnum verbosity = VERBOSITY_NORMAL; + +/* defined as static so that die() can close them */ +static PGconn *subscriber_conn = NULL; +static PGconn *provider_conn = NULL; + +static void signal_handler(int sig); +static void usage(void); +static void die(const char *fmt,...) +pg_attribute_printf(1, 2); +static void print_msg(VerbosityLevelEnum level, const char *fmt,...) +pg_attribute_printf(2, 3); + +static int run_pg_ctl(const char *arg); +static void run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args); +static void wait_postmaster_connection(const char *connstr); +static void wait_primary_connection(const char *connstr); +static void wait_postmaster_shutdown(void); + +static char *validate_replication_set_input(char *replication_sets); + +static void remove_unwanted_data(PGconn *conn); +static void initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn); +static char *create_restore_point(PGconn *conn, char *restore_point_name); +static char *initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists); +static void spock_subscribe(PGconn *conn, char *subscriber_name, + char *subscriber_dsn, + char *provider_connstr, + char *replication_sets, + int apply_delay, + bool force_text_transfer); + +static RemoteInfo *get_remote_info(PGconn* conn); + +static bool extension_exists(PGconn *conn, const char *extname); +static void install_extension(PGconn *conn, const char *extname); + +static void initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *pg_hba_conf, + char *extra_basebackup_args); +static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); + +static char *read_sysid(const char *data_dir); + +static void WriteRecoveryConf(PQExpBuffer contents); +static void CopyConfFile(char *fromfile, char *tofile, bool append); + +static char *get_connstr_dbname(char *connstr); +static char *get_connstr(char *connstr, char *dbname); +static char *PQconninfoParamsToConnstr(const char *const * keywords, const char *const * values); +static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); + +static bool file_exists(const char *path); +static bool is_pg_dir(const char *path); +static void copy_file(char *fromfile, char *tofile, bool append); +static char *find_other_exec_or_die(const char *argv0, const char *target); +static bool postmaster_is_alive(pid_t pid); +static long get_pgpid(void); +static char **get_database_list(char *databases, int *n_databases); +static char *generate_restore_point_name(void); + +static PGconn * +connectdb(const char *connstr) +{ + PGconn *conn; + + conn = PQconnectdb(connstr); + if (PQstatus(conn) != CONNECTION_OK) + die(_("Connection to database failed: %s, connection string was: %s\n"), PQerrorMessage(conn), connstr); + + return conn; +} + +void signal_handler(int sig) +{ + if (sig == SIGINT) + { + die(_("\nCanceling...\n")); + } +} + + +int +main(int argc, char **argv) +{ + int i; + int c; + PQExpBuffer recoveryconfcontents = createPQExpBuffer(); + RemoteInfo *remote_info; + char *remote_lsn; + bool stop = false; + bool drop_slot_if_exists = false; + int optindex; + char *subscriber_name = NULL; + char *base_sub_connstr = NULL; + char *base_prov_connstr = NULL; + char *replication_sets = NULL; + char *databases = NULL; + char *postgresql_conf = NULL, + *pg_hba_conf = NULL, + *recovery_conf = NULL; + int apply_delay = 0; + bool force_text_transfer = false; + char **slot_names; + char *sub_connstr; + char *prov_connstr; + char **database_list = { NULL }; + int n_databases = 1; + int dbnum; + bool use_existing_data_dir = false; + int pg_ctl_ret, + logfd; + char *restore_point_name = NULL; + char *extra_basebackup_args = NULL; + + static struct option long_options[] = { + {"subscriber-name", required_argument, NULL, 'n'}, + {"pgdata", required_argument, NULL, 'D'}, + {"provider-dsn", required_argument, NULL, 1}, + {"subscriber-dsn", required_argument, NULL, 2}, + {"replication-sets", required_argument, NULL, 3}, + {"postgresql-conf", required_argument, NULL, 4}, + {"hba-conf", required_argument, NULL, 5}, + {"recovery-conf", required_argument, NULL, 6}, + {"stop", no_argument, NULL, 's'}, + {"drop-slot-if-exists", no_argument, NULL, 7}, + {"apply-delay", required_argument, NULL, 8}, + {"databases", required_argument, NULL, 9}, + {"extra-basebackup-args", required_argument, NULL, 10}, + {"text-types", no_argument, NULL, 11}, + {NULL, 0, NULL, 0} + }; + + argv0 = argv[0]; + progname = get_progname(argv[0]); + start_time = time(NULL); + signal(SIGINT, signal_handler); + + /* check for --help */ + if (argc > 1) + { + for (i = 1; i < argc; i++) + { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-?") == 0) + { + usage(); + exit(0); + } + } + } + + /* Option parsing and validation */ + while ((c = getopt_long(argc, argv, "D:n:sv", long_options, &optindex)) != -1) + { + switch (c) + { + case 'D': + data_dir = pg_strdup(optarg); + break; + case 'n': + subscriber_name = pg_strdup(optarg); + break; + case 1: + base_prov_connstr = pg_strdup(optarg); + break; + case 2: + base_sub_connstr = pg_strdup(optarg); + break; + case 3: + replication_sets = validate_replication_set_input(pg_strdup(optarg)); + break; + case 4: + { + postgresql_conf = pg_strdup(optarg); + if (postgresql_conf != NULL && !file_exists(postgresql_conf)) + die(_("The specified postgresql.conf file does not exist.")); + break; + } + case 5: + { + pg_hba_conf = pg_strdup(optarg); + if (pg_hba_conf != NULL && !file_exists(pg_hba_conf)) + die(_("The specified pg_hba.conf file does not exist.")); + break; + } + case 6: + { + recovery_conf = pg_strdup(optarg); + if (recovery_conf != NULL && !file_exists(recovery_conf)) + die(_("The specified recovery configuration file does not exist.")); + break; + } + case 'v': + verbosity++; + break; + case 's': + stop = true; + break; + case 7: + drop_slot_if_exists = true; + break; + case 8: + apply_delay = atoi(optarg); + break; + case 9: + databases = pg_strdup(optarg); + break; + case 10: + extra_basebackup_args = pg_strdup(optarg); + break; + case 11: + force_text_transfer = true; + break; + default: + fprintf(stderr, _("Unknown option\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + } + + /* + * Sanity checks + */ + + if (data_dir == NULL) + { + fprintf(stderr, _("No data directory specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + else if (subscriber_name == NULL) + { + fprintf(stderr, _("No subscriber name specified\n")); + fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + exit(1); + } + + if (!base_prov_connstr || !strlen(base_prov_connstr)) + die(_("Provider connection string must be specified.\n")); + if (!base_sub_connstr || !strlen(base_sub_connstr)) + die(_("Subscriber connection string must be specified.\n")); + + if (apply_delay < 0) + die(_("Apply delay cannot be negative.\n")); + + if (apply_delay > MAX_APPLY_DELAY) + die(_("Apply delay cannot be more than %d.\n"), MAX_APPLY_DELAY); + + if (!replication_sets || !strlen(replication_sets)) + replication_sets = "default,default_insert_only,ddl_sql"; + + /* Init random numbers used for slot suffixes, etc */ + srand(time(NULL)); + + /* Parse database list or connection string. */ + if (databases != NULL) + { + database_list = get_database_list(databases, &n_databases); + } + else + { + char *dbname = get_connstr_dbname(base_prov_connstr); + + if (!dbname) + die(_("Either provider connection string must contain database " + "name or --databases option must be specified.\n")); + + n_databases = 1; + database_list = palloc(n_databases * sizeof(char *)); + database_list[0] = dbname; + } + + slot_names = palloc(n_databases * sizeof(char *)); + + /* + * Check connection strings for validity before doing anything + * expensive. + */ + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + prov_connstr = get_connstr(base_prov_connstr, db); + if (!prov_connstr || !strlen(prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + sub_connstr = get_connstr(base_sub_connstr, db); + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } + + /* + * Create log file where new postgres instance will log to while being + * initialized. + */ + logfd = open("spock_create_subscriber_postgres.log", O_CREAT | O_RDWR, + S_IRUSR | S_IWUSR); + if (logfd == -1) + { + die(_("Creating spock_create_subscriber_postgres.log failed: %s"), + strerror(errno)); + } + /* Safe to close() unchecked, we didn't write */ + (void) close(logfd); + + /* Let's start the real work... */ + print_msg(VERBOSITY_NORMAL, _("%s: starting ...\n"), progname); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + prov_connstr = get_connstr(base_prov_connstr, db); + if (!prov_connstr || !strlen(prov_connstr)) + die(_("Provider connection string is not valid.\n")); + + /* Read the remote server indetification. */ + print_msg(VERBOSITY_NORMAL, + _("Getting information for database %s ...\n"), db); + provider_conn = connectdb(prov_connstr); + remote_info = get_remote_info(provider_conn); + + /* only need to do this piece once */ + + if (dbnum == 0) + { + use_existing_data_dir = check_data_dir(data_dir, remote_info); + + if (use_existing_data_dir && + strcmp(remote_info->sysid, read_sysid(data_dir)) != 0) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } + + /* + * Create replication slots on remote node. + */ + print_msg(VERBOSITY_NORMAL, + _("Creating replication slot in database %s ...\n"), db); + slot_names[dbnum] = initialize_replication_slot(provider_conn, + remote_info->dbname, + remote_info->node_name, + subscriber_name, + drop_slot_if_exists); + PQfinish(provider_conn); + provider_conn = NULL; + } + + /* + * Create basebackup or use existing one + */ + prov_connstr = get_connstr(base_prov_connstr, database_list[0]); + sub_connstr = get_connstr(base_sub_connstr, database_list[0]); + + initialize_data_dir(data_dir, + use_existing_data_dir ? NULL : prov_connstr, + postgresql_conf, pg_hba_conf, + extra_basebackup_args); + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + + restore_point_name = generate_restore_point_name(); + + print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), + restore_point_name); + provider_conn = connectdb(prov_connstr); + remote_lsn = create_restore_point(provider_conn, restore_point_name); + PQfinish(provider_conn); + provider_conn = NULL; + + /* + * Get subscriber db to consistent state (for lsn after slot creation). + */ + print_msg(VERBOSITY_NORMAL, + _("Bringing subscriber node to the restore point ...\n")); + if (recovery_conf) + { + CopyConfFile(recovery_conf, "postgresql.auto.conf", true); + } + else + { + appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", + escape_single_quotes_ascii(prov_connstr)); + } + appendPQExpBuffer(recoveryconfcontents, "recovery_target_name = '%s'\n", restore_point_name); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_inclusive = true\n"); + appendPQExpBuffer(recoveryconfcontents, "recovery_target_action = promote\n"); + WriteRecoveryConf(recoveryconfcontents); + + free(restore_point_name); + restore_point_name = NULL; + + /* + * Start subscriber node with spock disabled, and wait until it starts + * accepting connections which means it has caught up to the restore point. + */ + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + + wait_primary_connection(sub_connstr); + + /* + * Clean any per-node data that were copied by pg_basebackup. + */ + print_msg(VERBOSITY_VERBOSE, + _("Removing old spock configuration ...\n")); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + + /* Stop Postgres so we can reset system id and start it with spock loaded. */ + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); + wait_postmaster_shutdown(); + + /* + * Start the node again, now with spock active so that we can start the + * logical replication. This is final start, so don't log to to special log + * file anymore. + */ + print_msg(VERBOSITY_NORMAL, + _("Initializing spock on the subscriber node:\n")); + + pg_ctl_ret = run_pg_ctl("start"); + if (pg_ctl_ret != 0) + die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); + wait_postmaster_connection(base_sub_connstr); + + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + prov_connstr = get_connstr(base_prov_connstr, db); + + subscriber_conn = connectdb(sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + if (PQserverVersion(subscriber_conn) < 90500) + install_extension(subscriber_conn, "spock_origin"); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which we + * already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + + spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, + prov_connstr, replication_sets, apply_delay, + force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + + /* If user does not want the node to be running at the end, stop it. */ + if (stop) + { + print_msg(VERBOSITY_NORMAL, _("Stopping the subscriber node ...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Stopping postgres after successful subscribtion failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, _("All done\n")); + + return 0; +} + + +/* + * Print help. + */ +static void +usage(void) +{ + printf(_("%s create new spock subscriber from basebackup of provider.\n\n"), progname); + printf(_("Usage:\n")); + printf(_(" %s [OPTION]...\n"), progname); + printf(_("\nGeneral options:\n")); + printf(_(" -D, --pgdata=DIRECTORY data directory to be used for new node,\n")); + printf(_(" can be either empty/non-existing directory,\n")); + printf(_(" or directory populated using\n")); + printf(_(" pg_basebackup -X stream command\n")); + printf(_(" --databases optional list of databases to replicate\n")); + printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber\n")); + printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); + printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); + printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); + printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); + printf(_(" -s, --stop stop the server once the initialization is done\n")); + printf(_(" -v increase logging verbosity\n")); + printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); + printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_("\nConfiguration files override:\n")); + printf(_(" --hba-conf path to the new pg_hba.conf\n")); + printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --recovery-conf path to the template recovery configuration\n")); +} + +/* + * Print error and exit. + */ +static void +die(const char *fmt,...) +{ + va_list argptr; + va_start(argptr, fmt); + vfprintf(stderr, fmt, argptr); + va_end(argptr); + + if (subscriber_conn) + PQfinish(subscriber_conn); + if (provider_conn) + PQfinish(provider_conn); + + if (get_pgpid()) + { + if (!run_pg_ctl("stop -s")) + { + fprintf(stderr, _("WARNING: postgres seems to be running, but could not be stopped\n")); + } + } + + exit(1); +} + +/* + * Print message to stdout and flush + */ +static void +print_msg(VerbosityLevelEnum level, const char *fmt,...) +{ + if (verbosity >= level) + { + va_list argptr; + va_start(argptr, fmt); + vfprintf(stdout, fmt, argptr); + va_end(argptr); + fflush(stdout); + } +} + + +/* + * Start pg_ctl with given argument(s) - used to start/stop postgres + * + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * signal this call will die and not return. + */ +static int +run_pg_ctl(const char *arg) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); + + appendPQExpBuffer(cmd, "%s %s -D \"%s\"", exec_path, arg, data_dir); + + /* Run pg_ctl in silent mode unless we run in debug mode. */ + if (verbosity < VERBOSITY_DEBUG) + appendPQExpBuffer(cmd, " -s"); + + print_msg(VERBOSITY_DEBUG, _("Running pg_ctl: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret)) + return WEXITSTATUS(ret); + else if (WIFSIGNALED(ret)) + die(_("pg_ctl exited with signal %d"), WTERMSIG(ret)); + else + die(_("pg_ctl exited for an unknown reason (system() returned %d)"), ret); + + return -1; +} + + +/* + * Run pg_basebackup to create the copy of the origin node. + */ +static void +run_basebackup(const char *provider_connstr, const char *data_dir, + const char *extra_basebackup_args) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_basebackup"); + + appendPQExpBuffer(cmd, "%s -D \"%s\" -d \"%s\" -X s -P", exec_path, data_dir, provider_connstr); + + /* Run pg_basebackup in verbose mode if we are running in verbose mode. */ + if (verbosity >= VERBOSITY_VERBOSE) + appendPQExpBuffer(cmd, " -v"); + + if (extra_basebackup_args != NULL) + appendPQExpBuffer(cmd, "%s", extra_basebackup_args); + + print_msg(VERBOSITY_DEBUG, _("Running pg_basebackup: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 0) + return; + if (WIFEXITED(ret)) + die(_("pg_basebackup failed with exit status %d, cannot continue.\n"), WEXITSTATUS(ret)); + else if (WIFSIGNALED(ret)) + die(_("pg_basebackup exited with signal %d, cannot continue"), WTERMSIG(ret)); + else + die(_("pg_basebackup exited for an unknown reason (system() returned %d)"), ret); +} + +/* + * Init the datadir + * + * This function can either ensure provided datadir is a postgres datadir, + * or create it using pg_basebackup. + * + * In any case, new postresql.conf and pg_hba.conf will be copied to the + * datadir if they are provided. + */ +static void +initialize_data_dir(char *data_dir, char *connstr, + char *postgresql_conf, char *pg_hba_conf, + char *extra_basebackup_args) +{ + if (connstr) + { + print_msg(VERBOSITY_NORMAL, + _("Creating base backup of the remote node...\n")); + run_basebackup(connstr, data_dir, extra_basebackup_args); + } + + if (postgresql_conf) + CopyConfFile(postgresql_conf, "postgresql.conf", false); + if (pg_hba_conf) + CopyConfFile(pg_hba_conf, "pg_hba.conf", false); +} + +/* + * This function checks if provided datadir is clone of the remote node + * described by the remote info, or if it's emtpy directory that can be used + * as new datadir. + */ +static bool +check_data_dir(char *data_dir, RemoteInfo *remoteinfo) +{ + /* Run basebackup as needed. */ + switch (pg_check_dir(data_dir)) + { + case 0: /* Does not exist */ + case 1: /* Exists, empty */ + return false; + case 2: + case 3: /* Exists, not empty */ + case 4: + { + if (!is_pg_dir(data_dir)) + die(_("Directory \"%s\" exists but is not valid postgres data directory.\n"), + data_dir); + return true; + } + case -1: /* Access problem */ + die(_("Could not access directory \"%s\": %s.\n"), + data_dir, strerror(errno)); + } + + /* Unreachable */ + die(_("Unexpected result from pg_check_dir() call")); + return false; +} + +/* + * Initialize replication slots + */ +static char * +initialize_replication_slot(PGconn *conn, char *dbname, + char *provider_node_name, char *subscription_name, + bool drop_slot_if_exists) +{ + PQExpBufferData query; + char *slot_name; + PGresult *res; + + /* Generate the slot name. */ + initPQExpBuffer(&query); + printfPQExpBuffer(&query, + "SELECT spock.spock_gen_slot_name(%s, %s, %s)", + PQescapeLiteral(conn, dbname, strlen(dbname)), + PQescapeLiteral(conn, provider_node_name, + strlen(provider_node_name)), + PQescapeLiteral(conn, subscription_name, + strlen(subscription_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could generate slot name: %s"), PQerrorMessage(conn)); + + slot_name = pstrdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + resetPQExpBuffer(&query); + + /* Check if the current slot exists. */ + printfPQExpBuffer(&query, + "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could not fetch existing slot information: %s"), PQerrorMessage(conn)); + + /* Drop the existing slot when asked for it or error if it already exists. */ + if (PQntuples(res) > 0) + { + PQclear(res); + resetPQExpBuffer(&query); + + if (!drop_slot_if_exists) + die(_("Slot %s already exists, drop it or use --drop-slot-if-exists to drop it automatically.\n"), + slot_name); + + print_msg(VERBOSITY_VERBOSE, + _("Droping existing slot %s ...\n"), slot_name); + + printfPQExpBuffer(&query, + "SELECT pg_catalog.pg_drop_replication_slot(%s)", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("Could not drop existing slot %s: %s"), slot_name, + PQerrorMessage(conn)); + } + + PQclear(res); + resetPQExpBuffer(&query); + + /* And finally, create the slot. */ + appendPQExpBuffer(&query, "SELECT pg_create_logical_replication_slot(%s, '%s');", + PQescapeLiteral(conn, slot_name, strlen(slot_name)), + "spock_output"); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication slot, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + + PQclear(res); + termPQExpBuffer(&query); + + return slot_name; +} + +/* + * Read replication info about remote connection + * + * TODO: unify with spock_remote_node_info in spock_rpc + */ +static RemoteInfo * +get_remote_info(PGconn* conn) +{ + RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); + PGresult *res; + + if (!extension_exists(conn, "spock")) + die(_("The remote node is not configured as a spock provider.\n")); + + res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); + + /* No nodes found? */ + if (PQntuples(res) == 0) + die(_("The remote database is not configured as a spock node.\n")); + + if (PQntuples(res) > 1) + die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + + ri->nodeid = atooid(PQgetvalue(res, 0, 0)); + ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); + ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); + ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); + ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + + PQclear(res); + + return ri; +} + +/* + * Check if extension exists. + */ +static bool +extension_exists(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + bool ret; + + printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", + PQescapeLiteral(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); + } + + ret = PQntuples(res) == 1; + + PQclear(res); + destroyPQExpBuffer(query); + + return ret; +} + +/* + * Create extension. + */ +static void +install_extension(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", + PQescapeIdentifier(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + PQclear(res); + die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); + } + + PQclear(res); + destroyPQExpBuffer(query); +} + +/* + * Clean all the data that was copied from remote node but we don't + * want it here (currently shared security labels and replication identifiers). + */ +static void +remove_unwanted_data(PGconn *conn) +{ + PGresult *res; + + /* + * Remove replication identifiers (9.4 will get them removed by dropping + * the extension later as we emulate them there). + */ + if (PQserverVersion(conn) >= 90500) + { + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + } + PQclear(res); + } + + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); +} + +/* + * Initialize new remote identifier to specific position. + */ +static void +initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) +{ + PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + + if (PQserverVersion(conn) >= 90500) + { + printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", + PQescapeLiteral(conn, origin_name, strlen(origin_name))); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + if (remote_lsn) + { + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not advance replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + } + } + else + { + printfPQExpBuffer(query, "INSERT INTO spock_origin.replication_origin (roident, roname, roremote_lsn) SELECT COALESCE(MAX(roident::int), 0) + 1, %s, %s FROM spock_origin.replication_origin", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn ? PQescapeLiteral(conn, remote_lsn, strlen(remote_lsn)) : "0"); + + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + } + + destroyPQExpBuffer(query); +} + + +/* + * Create remote restore point which will be used to get into synchronized + * state through physical replay. + */ +static char * +create_restore_point(PGconn *conn, char *restore_point_name) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *remote_lsn = NULL; + + printfPQExpBuffer(query, "SELECT pg_create_restore_point('%s')", restore_point_name); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create restore point, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + remote_lsn = pstrdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + destroyPQExpBuffer(query); + + return remote_lsn; +} + +static void +spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, + char *provider_dsn, char *replication_sets, + int apply_delay, bool force_text_transfer) +{ + PQExpBufferData query; + PQExpBufferData repsets; + PGresult *res; + + initPQExpBuffer(&query); + printfPQExpBuffer(&query, + "SELECT spock.node_create(node_name := %s, dsn := %s);", + PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), + PQescapeLiteral(conn, subscriber_dsn, strlen(subscriber_dsn))); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create local node, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + resetPQExpBuffer(&query); + initPQExpBuffer(&repsets); + + printfPQExpBuffer(&repsets, "{%s}", replication_sets); + printfPQExpBuffer(&query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "apply_delay := '%d seconds'::interval, " + "synchronize_structure := false, " + "synchronize_data := false, " + "force_text_transfer := '%s');", + PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), + PQescapeLiteral(conn, provider_dsn, strlen(provider_dsn)), + PQescapeLiteral(conn, repsets.data, repsets.len), + apply_delay, (force_text_transfer ? "t" : "f")); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create subscription, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + /* TODO */ + res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not update subscription, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + + PQclear(res); + + termPQExpBuffer(&repsets); + termPQExpBuffer(&query); +} + + +/* + * Validates input of the replication sets and returns normalized data. + */ +static char * +validate_replication_set_input(char *replication_sets) +{ + char *name; + PQExpBuffer retbuf = createPQExpBuffer(); + char *ret; + bool first = true; + + if (!replication_sets) + return NULL; + + name = strtok(replication_sets, " ,"); + while (name != NULL) + { + const char *cp; + + if (strlen(name) == 0) + die(_("Replication set name \"%s\" is too short\n"), name); + + if (strlen(name) > NAMEDATALEN) + die(_("Replication set name \"%s\" is too long\n"), name); + + for (cp = name; *cp; cp++) + { + if (!((*cp >= 'a' && *cp <= 'z') + || (*cp >= '0' && *cp <= '9') + || (*cp == '_') + || (*cp == '-'))) + { + die(_("Replication set name \"%s\" contains invalid character\n"), + name); + } + } + + if (first) + first = false; + else + appendPQExpBufferStr(retbuf, ", "); + appendPQExpBufferStr(retbuf, name); + + name = strtok(NULL, " ,"); + } + + ret = pg_strdup(retbuf->data); + destroyPQExpBuffer(retbuf); + + return ret; +} + +static char * +get_connstr_dbname(char *connstr) +{ + PQconninfoOption *conn_opts = NULL; + PQconninfoOption *conn_opt; + char *err_msg = NULL; + char *ret = NULL; + + conn_opts = PQconninfoParse(connstr, &err_msg); + if (conn_opts == NULL) + { + die(_("Invalid connection string: %s\n"), err_msg); + } + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (strcmp(conn_opt->keyword, "dbname") == 0) + { + ret = pstrdup(conn_opt->val); + break; + } + } + + PQconninfoFree(conn_opts); + + return ret; +} + + +/* + * Build connection string from individual parameter. + * + * dbname can be specified in connstr parameter + */ +static char * +get_connstr(char *connstr, char *dbname) +{ + char *ret; + int argcount = 4; /* dbname, host, user, port */ + int i; + const char **keywords; + const char **values; + PQconninfoOption *conn_opts = NULL; + PQconninfoOption *conn_opt; + char *err_msg = NULL; + + /* + * Merge the connection info inputs given in form of connection string + * and options + */ + i = 0; + if (connstr && + (strncmp(connstr, "postgresql://", 13) == 0 || + strncmp(connstr, "postgres://", 11) == 0 || + strchr(connstr, '=') != NULL)) + { + conn_opts = PQconninfoParse(connstr, &err_msg); + if (conn_opts == NULL) + { + die(_("Invalid connection string: %s\n"), err_msg); + } + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + if (conn_opt->val != NULL && conn_opt->val[0] != '\0') + argcount++; + } + + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++) + { + /* If db* parameters were provided, we'll fill them later. */ + if (dbname && strcmp(conn_opt->keyword, "dbname") == 0) + continue; + + if (conn_opt->val != NULL && conn_opt->val[0] != '\0') + { + keywords[i] = conn_opt->keyword; + values[i] = conn_opt->val; + i++; + } + } + } + else + { + keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); + values = pg_malloc0((argcount + 1) * sizeof(*values)); + + /* + * If connstr was provided but it's not in connection string format and + * the dbname wasn't provided then connstr is actually dbname. + */ + if (connstr && !dbname) + dbname = connstr; + } + + if (dbname) + { + keywords[i] = "dbname"; + values[i] = dbname; + i++; + } + + ret = PQconninfoParamsToConnstr(keywords, values); + + /* Connection ok! */ + pg_free(values); + pg_free(keywords); + if (conn_opts) + PQconninfoFree(conn_opts); + + return ret; +} + + +/* + * Reads the pg_control file of the existing data dir. + */ +static char * +read_sysid(const char *data_dir) +{ + ControlFileData ControlFile; + int fd; + char ControlFilePath[MAXPGPATH]; + char *res = (char *) pg_malloc0(33); + + snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", data_dir); + + if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) + die(_("%s: could not open file \"%s\" for reading: %s\n"), + progname, ControlFilePath, strerror(errno)); + + if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) + die(_("%s: could not read file \"%s\": %s\n"), + progname, ControlFilePath, strerror(errno)); + + close(fd); + + snprintf(res, 33, UINT64_FORMAT, ControlFile.system_identifier); + return res; +} + +/* + * Write contents of recovery.conf or postgresql.auto.conf + */ +static void +WriteRecoveryConf(PQExpBuffer contents) +{ + char filename[MAXPGPATH]; + FILE *cf; + + sprintf(filename, "%s/postgresql.auto.conf", data_dir); + + cf = fopen(filename, "a"); + if (cf == NULL) + { + die(_("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno)); + } + + if (fwrite(contents->data, contents->len, 1, cf) != 1) + { + die(_("%s: could not write to file \"%s\": %s\n"), + progname, filename, strerror(errno)); + } + + fclose(cf); + + { + sprintf(filename, "%s/standby.signal", data_dir); + cf = fopen(filename, "w"); + if (cf == NULL) + { + die(_("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno)); + } + + fclose(cf); + } +} + +/* + * Copy file to data + */ +static void +CopyConfFile(char *fromfile, char *tofile, bool append) +{ + char filename[MAXPGPATH]; + + sprintf(filename, "%s/%s", data_dir, tofile); + + print_msg(VERBOSITY_DEBUG, _("Copying \"%s\" to \"%s\".\n"), + fromfile, filename); + copy_file(fromfile, filename, append); +} + + +/* + * Convert PQconninfoOption array into conninfo string + */ +static char * +PQconninfoParamsToConnstr(const char *const * keywords, const char *const * values) +{ + PQExpBuffer retbuf = createPQExpBuffer(); + char *ret; + int i = 0; + + for (i = 0; keywords[i] != NULL; i++) + { + if (i > 0) + appendPQExpBufferChar(retbuf, ' '); + appendPQExpBuffer(retbuf, "%s=", keywords[i]); + appendPQExpBufferConnstrValue(retbuf, values[i]); + } + + ret = pg_strdup(retbuf->data); + destroyPQExpBuffer(retbuf); + + return ret; +} + +/* + * Escape connection info value + */ +static void +appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) +{ + const char *s; + bool needquotes; + + /* + * If the string consists entirely of plain ASCII characters, no need to + * quote it. This is quite conservative, but better safe than sorry. + */ + needquotes = false; + for (s = str; *s; s++) + { + if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || + (*s >= '0' && *s <= '9') || *s == '_' || *s == '.')) + { + needquotes = true; + break; + } + } + + if (needquotes) + { + appendPQExpBufferChar(buf, '\''); + while (*str) + { + /* ' and \ must be escaped by to \' and \\ */ + if (*str == '\'' || *str == '\\') + appendPQExpBufferChar(buf, '\\'); + + appendPQExpBufferChar(buf, *str); + str++; + } + appendPQExpBufferChar(buf, '\''); + } + else + appendPQExpBufferStr(buf, str); +} + + +/* + * Find the pgport and try a connection + */ +static void +wait_postmaster_connection(const char *connstr) +{ + PGPing res; + long pmpid = 0; + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to accept connections ..."); + + /* First wait for Postmaster to come up. */ + for (;;) + { + if ((pmpid = get_pgpid()) != 0 && + postmaster_is_alive((pid_t) pmpid)) + break; + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + /* Now wait for Postmaster to either accept connections or die. */ + for (;;) + { + res = PQping(connstr); + if (res == PQPING_OK) + break; + else if (res == PQPING_NO_ATTEMPT) + break; + + /* + * Check if the process is still alive. This covers cases where the + * postmaster successfully created the pidfile but then crashed without + * removing it. + */ + if (!postmaster_is_alive((pid_t) pmpid)) + break; + + /* No response; wait */ + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + print_msg(VERBOSITY_VERBOSE, "\n"); +} + + +/* + * Wait for PostgreSQL to leave recovery/standby mode + */ +static void +wait_primary_connection(const char *connstr) +{ + bool ispri = false; + PGconn *conn = NULL; + PGresult *res; + + wait_postmaster_connection(connstr); + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to become primary..."); + + while (!ispri) + { + if (!conn || PQstatus(conn) != CONNECTION_OK) + { + if (conn) + PQfinish(conn); + wait_postmaster_connection(connstr); + conn = connectdb(connstr); + } + + res = PQexec(conn, "SELECT pg_is_in_recovery()"); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') + ispri = true; + else + { + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + PQclear(res); + } + + PQfinish(conn); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +/* + * Wait for postmaster to die + */ +static void +wait_postmaster_shutdown(void) +{ + long pid; + + print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to shutdown ..."); + + for (;;) + { + if ((pid = get_pgpid()) != 0) + { + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_NORMAL, "."); + } + else + break; + } + + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +static bool +file_exists(const char *path) +{ + struct stat statbuf; + + if (stat(path, &statbuf) != 0) + return false; + + return true; +} + +static bool +is_pg_dir(const char *path) +{ + struct stat statbuf; + char version_file[MAXPGPATH]; + + if (stat(path, &statbuf) != 0) + return false; + + snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", data_dir); + if (stat(version_file, &statbuf) != 0 && errno == ENOENT) + { + return false; + } + + return true; +} + +/* + * copy one file + */ +static void +copy_file(char *fromfile, char *tofile, bool append) +{ + char *buffer; + int srcfd; + int dstfd; + int nbytes; + +#define COPY_BUF_SIZE (8 * BLCKSZ) + + buffer = malloc(COPY_BUF_SIZE); + + /* + * Open the files + */ + srcfd = open(fromfile, O_RDONLY | PG_BINARY, 0); + if (srcfd < 0) + die(_("could not open file \"%s\""), fromfile); + + dstfd = open(tofile, O_RDWR | O_CREAT | (append ? O_APPEND : O_TRUNC) | PG_BINARY, + S_IRUSR | S_IWUSR); + if (dstfd < 0) + die(_("could not create file \"%s\""), tofile); + + /* + * Do the data copying. + */ + for (;;) + { + nbytes = read(srcfd, buffer, COPY_BUF_SIZE); + if (nbytes < 0) + die(_("could not read file \"%s\""), fromfile); + if (nbytes == 0) + break; + errno = 0; + if ((int) write(dstfd, buffer, nbytes) != nbytes) + { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + die(_("could not write to file \"%s\""), tofile); + } + } + + if (close(dstfd)) + die(_("could not close file \"%s\""), tofile); + + /* we don't care about errors here */ + close(srcfd); + + free(buffer); +} + + +static char * +find_other_exec_or_die(const char *argv0, const char *target) +{ + int ret; + char *found_path; + uint32 bin_version; + + found_path = pg_malloc(MAXPGPATH); + + ret = find_other_exec_version(argv0, target, &bin_version, found_path); + + if (ret < 0) + { + char full_path[MAXPGPATH]; + + if (find_my_exec(argv0, full_path) < 0) + strlcpy(full_path, progname, sizeof(full_path)); + + if (ret == -1) + die(_("The program \"%s\" is needed by %s " + "but was not found in the\n" + "same directory as \"%s\".\n" + "Check your installation.\n"), + target, progname, full_path); + else + die(_("The program \"%s\" was found by \"%s\"\n" + "but was not the same version as %s.\n" + "Check your installation.\n"), + target, full_path, progname); + } + else + { + char full_path[MAXPGPATH]; + + if (find_my_exec(argv0, full_path) < 0) + strlcpy(full_path, progname, sizeof(full_path)); + + if (bin_version / 100 != PG_VERSION_NUM / 100) + die(_("The program \"%s\" was found by \"%s\"\n" + "but was not the same version as %s.\n" + "Check your installation.\n"), + target, full_path, progname); + + } + + return found_path; +} + +static bool +postmaster_is_alive(pid_t pid) +{ + /* + * Test to see if the process is still there. Note that we do not + * consider an EPERM failure to mean that the process is still there; + * EPERM must mean that the given PID belongs to some other userid, and + * considering the permissions on $PGDATA, that means it's not the + * postmaster we are after. + * + * Don't believe that our own PID or parent shell's PID is the postmaster, + * either. (Windows hasn't got getppid(), though.) + */ + if (pid == getpid()) + return false; +#ifndef WIN32 + if (pid == getppid()) + return false; +#endif + if (kill(pid, 0) == 0) + return true; + return false; +} + +static long +get_pgpid(void) +{ + FILE *pidf; + long pid; + + pidf = fopen(pid_file, "r"); + if (pidf == NULL) + { + return 0; + } + if (fscanf(pidf, "%ld", &pid) != 1) + { + return 0; + } + fclose(pidf); + return pid; +} + +static char ** +get_database_list(char *databases, int *n_databases) +{ + char *c; + char **result; + int num = 1; + for (c = databases; *c; c++ ) + if (*c == ',') + num++; + *n_databases = num; + result = palloc(num * sizeof(char *)); + num = 0; + /* clone the argument so we don't destroy it with strtok*/ + databases = pstrdup(databases); + c = strtok(databases, ","); + while (c != NULL) + { + result[num] = pstrdup(c); + num++; + c = strtok(NULL,","); + } + pfree(databases); + return result; +} + +static char * +generate_restore_point_name(void) +{ + char *rpn = malloc(NAMEDATALEN); + snprintf(rpn, NAMEDATALEN-1, "spock_create_subscriber_%lx", random()); + return rpn; +} From bc9ae7bd5612e5bb8c285c9873adb51a9a3168c2 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 02/14] spock_create_subscriber: drop legacy paths and extension usage --- .../spock_create_subscriber.c | 64 ++++++------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 8147c68b0..fbf85fdfe 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -531,8 +531,6 @@ main(int argc, char **argv) /* Create the extension. */ print_msg(VERBOSITY_VERBOSE, _("Creating spock extension for database %s...\n"), db); - if (PQserverVersion(subscriber_conn) < 90500) - install_extension(subscriber_conn, "spock_origin"); install_extension(subscriber_conn, "spock"); /* @@ -965,16 +963,13 @@ remove_unwanted_data(PGconn *conn) * Remove replication identifiers (9.4 will get them removed by dropping * the extension later as we emulate them there). */ - if (PQserverVersion(conn) >= 90500) + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); - } PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); } + PQclear(res); res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) @@ -994,49 +989,30 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) PGresult *res; PQExpBuffer query = createPQExpBuffer(); - if (PQserverVersion(conn) >= 90500) - { - printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", - PQescapeLiteral(conn, origin_name, strlen(origin_name))); - - res = PQexec(conn, query->data); + printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", + PQescapeLiteral(conn, origin_name, strlen(origin_name))); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - die(_("Could not create replication origin \"%s\": status %s: %s\n"), - query->data, - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } - PQclear(res); - - if (remote_lsn) - { - printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", - PQescapeLiteral(conn, origin_name, strlen(origin_name)), - remote_lsn); - - res = PQexec(conn, query->data); + res = PQexec(conn, query->data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - die(_("Could not advance replication origin \"%s\": status %s: %s\n"), - query->data, - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } - PQclear(res); - } + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication origin \"%s\": status %s: %s\n"), + query->data, + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } - else + PQclear(res); + + if (remote_lsn) { - printfPQExpBuffer(query, "INSERT INTO spock_origin.replication_origin (roident, roname, roremote_lsn) SELECT COALESCE(MAX(roident::int), 0) + 1, %s, %s FROM spock_origin.replication_origin", - PQescapeLiteral(conn, origin_name, strlen(origin_name)), - remote_lsn ? PQescapeLiteral(conn, remote_lsn, strlen(remote_lsn)) : "0"); + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, '%s')", + PQescapeLiteral(conn, origin_name, strlen(origin_name)), + remote_lsn); res = PQexec(conn, query->data); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + if (PQresultStatus(res) != PGRES_TUPLES_OK) { - die(_("Could not create replication origin \"%s\": status %s: %s\n"), + die(_("Could not advance replication origin \"%s\": status %s: %s\n"), query->data, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } From 20a14ef78260bfdd8f1d83e9b9b3a5cb51a181e8 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 22 Jun 2026 14:06:38 +0500 Subject: [PATCH 03/14] Added usage document for spock_create_subscriber --- docs/creating_subscriber_nodes.md | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 docs/creating_subscriber_nodes.md diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md new file mode 100644 index 000000000..bc4622394 --- /dev/null +++ b/docs/creating_subscriber_nodes.md @@ -0,0 +1,37 @@ +## Creating a Subscriber Node with pg_basebackup + +Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. + +#### Synopsis: + + `spock_create_subscriber [OPTION]...` + +**Options** + +Specify the following options as needed. + +| Option | Description +|----------|------------- +| `-D`, `--pgdata=DIRECTORY` | The `data` directory to be used for new node. This can be either an empty/non-existing directory, or a directory populated using the `pg_basebackup -X stream` command. +| `--databases` | An optional list of databases to replicate. +| `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. +| `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. +| `--provider-dsn=CONNSTR` | A connection string to the provider. +| `--replication-sets=SETS` | A comma separated list of replication set names. +| `--apply-delay=DELAY` | The apply delay in seconds (by default 0). +| `--drop-slot-if-exists` | Drop replication slot of conflicting name. +| `-s`, `--stop` | Stop the server once the initialization is done. +| `-v` | Increase logging verbosity. +| `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` + +**Configuration files overrides** + +You can use the following options to override the location of the configuration files. + +| Option | Description +|----------|------------- +|`--hba-conf` | path to the new `pg_hba.conf` +| `--postgresql-conf` | path to the new `postgresql.conf` +| `--recovery-conf` | path to the template recovery configuration + +Unlike `spock.sub_create`'s other data sync options, this method of cloning ignores replication sets and copies all tables on all databases. However, it's often much faster, especially over high-bandwidth connections. From 051794507aa7eaa4bf1250e1754f971c236897a2 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Wed, 22 Jul 2026 10:35:51 +0500 Subject: [PATCH 04/14] spock_create_subscriber address review feedback Harden option parsing and cleanup unrelated to bidirectional join: - Reject --apply-delay values that aren't a clean integer instead of silently taking atoi()'s partial parse. - Validate --extra-basebackup-args against shell metacharacters before it is appended to a system() command string; the args are otherwise a command-injection vector. - Free the read_sysid() allocation on the existing-data-dir sysid check. - Fix a missing separator when appending --extra-basebackup-args to the pg_basebackup command line. - Replace sprintf with snprintf when writing postgresql.auto.conf. - Scope the post-sub_create sync_status fixup with a WHERE clause instead of unconditionally rewriting every row. - Document --text-types and a docs formatting nit. --- docs/creating_subscriber_nodes.md | 5 +- .../spock_create_subscriber.c | 63 +++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/docs/creating_subscriber_nodes.md b/docs/creating_subscriber_nodes.md index bc4622394..64528071f 100644 --- a/docs/creating_subscriber_nodes.md +++ b/docs/creating_subscriber_nodes.md @@ -2,7 +2,7 @@ Spock supports creating a subscriber node by cloning the provider with [`pg_basebackup`](https://www.postgresql.org/docs/current/app-pgbasebackup.html) and starting it as a Spock subscriber. Use the `spock_create_subscriber` utility (located in the `bin` directory of your pgEdge platform installation) to register the node. -#### Synopsis: +### Synopsis: `spock_create_subscriber [OPTION]...` @@ -17,12 +17,13 @@ Specify the following options as needed. | `-n`, `--subscriber-name=NAME` | The name of the newly created subscriber. | `--subscriber-dsn=CONNSTR` | A connection string to the newly created subscriber. | `--provider-dsn=CONNSTR` | A connection string to the provider. -| `--replication-sets=SETS` | A comma separated list of replication set names. +| `--replication-sets=SETS` | A comma-separated list of replication set names. | `--apply-delay=DELAY` | The apply delay in seconds (by default 0). | `--drop-slot-if-exists` | Drop replication slot of conflicting name. | `-s`, `--stop` | Stop the server once the initialization is done. | `-v` | Increase logging verbosity. | `--extra-basebackup-args` | Additional arguments to pass to `pg_basebackup`. Safe options are: `-T`, `-c`, `--xlogdir`/`--waldir` +| `--text-types` | Transfer all column values as text rather than binary during initial sync. Use this when provider and subscriber differ in endianness or type representation. **Configuration files overrides** diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index fbf85fdfe..7f7b76d5f 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -90,6 +90,7 @@ static void print_msg(VerbosityLevelEnum level, const char *fmt,...) pg_attribute_printf(2, 3); static int run_pg_ctl(const char *arg); +static void validate_extra_basebackup_args(const char *args); static void run_basebackup(const char *provider_connstr, const char *data_dir, const char *extra_basebackup_args); static void wait_postmaster_connection(const char *connstr); @@ -281,13 +282,19 @@ main(int argc, char **argv) drop_slot_if_exists = true; break; case 8: - apply_delay = atoi(optarg); + { + char *endptr; + apply_delay = (int) strtol(optarg, &endptr, 10); + if (*endptr != '\0' || endptr == optarg) + die(_("--apply-delay requires an integer value\n")); + } break; case 9: databases = pg_strdup(optarg); break; case 10: extra_basebackup_args = pg_strdup(optarg); + validate_extra_basebackup_args(extra_basebackup_args); break; case 11: force_text_transfer = true; @@ -407,9 +414,14 @@ main(int argc, char **argv) { use_existing_data_dir = check_data_dir(data_dir, remote_info); - if (use_existing_data_dir && - strcmp(remote_info->sysid, read_sysid(data_dir)) != 0) - die(_("Subscriber data directory is not basebackup of remote node.\n")); + if (use_existing_data_dir) + { + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } } /* @@ -597,6 +609,9 @@ usage(void) printf(_(" -v increase logging verbosity\n")); printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); + printf(_(" --text-types transfer column values as text rather than binary\n")); + printf(_(" (use when provider and subscriber differ in type\n")); + printf(_(" representation or endianness)\n")); printf(_("\nConfiguration files override:\n")); printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); @@ -682,6 +697,27 @@ run_pg_ctl(const char *arg) } +/* + * Reject --extra-basebackup-args values containing shell control + * characters. The args are appended to a system() command string, so + * semicolons, pipes, backticks, and similar metacharacters would allow + * arbitrary command injection. + */ +static void +validate_extra_basebackup_args(const char *args) +{ + const char *p; + + for (p = args; *p; p++) + { + if (*p == ';' || *p == '|' || *p == '&' || *p == '`' || + *p == '$' || *p == '(' || *p == ')' || + *p == '<' || *p == '>' || *p == '{' || *p == '}' || + *p == '\n' || *p == '\r') + die(_("--extra-basebackup-args contains unsafe shell characters\n")); + } +} + /* * Run pg_basebackup to create the copy of the origin node. */ @@ -700,7 +736,7 @@ run_basebackup(const char *provider_connstr, const char *data_dir, appendPQExpBuffer(cmd, " -v"); if (extra_basebackup_args != NULL) - appendPQExpBuffer(cmd, "%s", extra_basebackup_args); + appendPQExpBuffer(cmd, " %s", extra_basebackup_args); print_msg(VERBOSITY_DEBUG, _("Running pg_basebackup: %s.\n"), cmd->data); ret = system(cmd->data); @@ -1097,8 +1133,8 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, } PQclear(res); - /* TODO */ - res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'"); + res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" + " WHERE sync_status != 'r'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not update subscription, status %s: %s\n"), @@ -1317,7 +1353,7 @@ WriteRecoveryConf(PQExpBuffer contents) char filename[MAXPGPATH]; FILE *cf; - sprintf(filename, "%s/postgresql.auto.conf", data_dir); + snprintf(filename, sizeof(filename), "%s/postgresql.auto.conf", data_dir); cf = fopen(filename, "a"); if (cf == NULL) @@ -1334,7 +1370,7 @@ WriteRecoveryConf(PQExpBuffer contents) fclose(cf); { - sprintf(filename, "%s/standby.signal", data_dir); + snprintf(filename, sizeof(filename), "%s/standby.signal", data_dir); cf = fopen(filename, "w"); if (cf == NULL) { @@ -1353,7 +1389,7 @@ CopyConfFile(char *fromfile, char *tofile, bool append) { char filename[MAXPGPATH]; - sprintf(filename, "%s/%s", data_dir, tofile); + snprintf(filename, sizeof(filename), "%s/%s", data_dir, tofile); print_msg(VERBOSITY_DEBUG, _("Copying \"%s\" to \"%s\".\n"), fromfile, filename); @@ -1560,7 +1596,7 @@ is_pg_dir(const char *path) if (stat(path, &statbuf) != 0) return false; - snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", data_dir); + snprintf(version_file, MAXPGPATH, "%s/PG_VERSION", path); if (stat(version_file, &statbuf) != 0 && errno == ENOENT) { return false; @@ -1711,6 +1747,7 @@ get_pgpid(void) } if (fscanf(pidf, "%ld", &pid) != 1) { + fclose(pidf); return 0; } fclose(pidf); @@ -1746,6 +1783,8 @@ static char * generate_restore_point_name(void) { char *rpn = malloc(NAMEDATALEN); - snprintf(rpn, NAMEDATALEN-1, "spock_create_subscriber_%lx", random()); + if (rpn == NULL) + die(_("out of memory\n")); + snprintf(rpn, NAMEDATALEN, "spock_create_subscriber_%lx", random()); return rpn; } From 361ee4df75f9766b49e9be11be247f61e9a6dac1 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 9 Jul 2026 15:43:36 +0500 Subject: [PATCH 05/14] spock_create_subscriber: add bidirectional join plumbing Add --bidirectional, --stall-timeout, --max-wait, and --cleanup options to support the bidirectional node-join procedure defined in the SPOC-601 design. In --bidirectional mode the tool connects to the source cluster, discovers all peer nodes via spock.subscription/node/node_interface, verifies preconditions (Spock >= 6.0.0, track_commit_timestamp on on all nodes, no pending DDL, full-mesh topology, per-peer connectivity), then writes a JSON manifest to /spock_bidirectional_manifest.json and exits. The manifest records peer names, DSNs, slot names, and sub names so that later phases can resume idempotently. In --cleanup mode the tool reads the manifest and idempotently removes any partial state left by a prior attempt: drops replication slots on the source and each peer, drops reverse subscriptions, and removes the manifest file. Connectivity failures during cleanup are logged as warnings rather than being fatal. No replication behavior is changed in this commit; the subscriber DSN is not required in --bidirectional mode since the subscriber database does not exist yet at this stage. --- .../spock_create_subscriber.c | 779 +++++++++++++++++- 1 file changed, 773 insertions(+), 6 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 7f7b76d5f..bff537361 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -52,6 +52,8 @@ #include "access/timeline.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" +#include "common/jsonapi.h" +#include "mb/pg_wchar.h" #include "spock_fe.h" @@ -65,6 +67,30 @@ typedef struct RemoteInfo { char *replication_sets; } RemoteInfo; +typedef struct PeerNodeInfo +{ + char *node_name; + char *dsn; + char *slot_name; /* from spock.spock_gen_slot_name() */ + char *sub_name; /* "sub__" */ + bool disabled_sub_created; + bool slot_created; + bool reverse_sub_created; +} PeerNodeInfo; + +typedef struct BidirectionalState +{ + bool enabled; + int num_peers; + PeerNodeInfo *peers; + int stall_timeout; /* default 600s */ + int max_wait; /* default 0 = unbounded */ + char *source_slot_name; + char *source_origin_name; + bool cleanup_mode; + char *manifest_path; +} BidirectionalState; + typedef enum { VERBOSITY_NORMAL, VERBOSITY_VERBOSE, @@ -141,6 +167,20 @@ static long get_pgpid(void); static char **get_database_list(char *databases, int *n_databases); static char *generate_restore_point_name(void); +static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out); +static void check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers); +static void write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn); +static bool read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out); +static void cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir); +static void append_json_string(PQExpBuffer buf, const char *str); + static PGconn * connectdb(const char *connstr) { @@ -161,6 +201,654 @@ void signal_handler(int sig) } } +/* + * Append str to buf with JSON string escaping applied, without the + * surrounding quotes (the caller supplies those). Control characters + * below 0x20 are emitted as \uXXXX. jsonapi.h provides a JSON parser but + * no encoder, so this is a small local encoder in the same style as + * src/bin/pg_combinebackup/write_manifest.c. + */ +static void +append_json_string(PQExpBuffer buf, const char *str) +{ + const char *p; + + for (p = str; *p; p++) + { + switch (*p) + { + case '\b': appendPQExpBufferStr(buf, "\\b"); break; + case '\f': appendPQExpBufferStr(buf, "\\f"); break; + case '\n': appendPQExpBufferStr(buf, "\\n"); break; + case '\r': appendPQExpBufferStr(buf, "\\r"); break; + case '\t': appendPQExpBufferStr(buf, "\\t"); break; + case '"': appendPQExpBufferStr(buf, "\\\""); break; + case '\\': appendPQExpBufferStr(buf, "\\\\"); break; + default: + if ((unsigned char) *p < 0x20) + appendPQExpBuffer(buf, "\\u%04x", (unsigned char) *p); + else + appendPQExpBufferChar(buf, *p); + break; + } + } +} + +/* + * Query the source for all peer nodes in the multi-master cluster. + * Returns the peer count; *peers_out is set to a pg_malloc0'd array. For + * each peer, sub_name is derived as "sub__" + * and slot_name is obtained via spock.spock_gen_slot_name() on the source. + */ +static int +discover_peer_nodes(PGconn *source_conn, const char *source_node_name, + const char *subscriber_name, const char *dbname, + PeerNodeInfo **peers_out) +{ + static const char *discover_sql = + "SELECT DISTINCT n.node_name, ni.if_dsn" + " FROM spock.subscription s" + " JOIN spock.node n ON s.sub_origin = n.node_id" + " JOIN spock.node_interface ni ON n.node_id = ni.if_nodeid" + " WHERE n.node_name != $1" + " ORDER BY n.node_name"; + const char *paramValues[3]; + PGresult *res; + PGresult *slot_res; + int npeers; + PeerNodeInfo *peers; + int i; + + paramValues[0] = source_node_name; + res = PQexecParams(source_conn, discover_sql, + 1, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not discover peer nodes: %s"), + PQerrorMessage(source_conn)); + + npeers = PQntuples(res); + if (npeers == 0) + { + PQclear(res); + die(_("no peer nodes found; source does not appear to be part of a " + "multi-master cluster")); + } + + peers = pg_malloc0(npeers * sizeof(PeerNodeInfo)); + + for (i = 0; i < npeers; i++) + { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + + peers[i].node_name = pg_strdup(PQgetvalue(res, i, 0)); + peers[i].dsn = pg_strdup(PQgetvalue(res, i, 1)); + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, peers[i].node_name); + peers[i].sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + + paramValues[0] = dbname; + paramValues[1] = peers[i].node_name; + paramValues[2] = peers[i].sub_name; + slot_res = PQexecParams(source_conn, + "SELECT spock.spock_gen_slot_name" + "($1::name, $2::name, $3::name)", + 3, NULL, paramValues, NULL, NULL, 0); + if (PQresultStatus(slot_res) != PGRES_TUPLES_OK) + die(_("could not generate slot name for peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(source_conn)); + + peers[i].slot_name = pg_strdup(PQgetvalue(slot_res, 0, 0)); + PQclear(slot_res); + + print_msg(VERBOSITY_VERBOSE, + _(" discovered peer: %s (slot: %s)\n"), + peers[i].node_name, peers[i].slot_name); + } + + PQclear(res); + *peers_out = peers; + return npeers; +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0, track_commit_timestamp on, no + * pending DDL, full-mesh topology, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + + /* Spock version gate: require >= 6.0.0 */ + res = PQexec(source_conn, + "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not query Spock extension version: %s"), + PQerrorMessage(source_conn)); + if (PQntuples(res) == 0) + die(_("Spock extension is not installed on the source node")); + { + const char *ver = PQgetvalue(res, 0, 0); + int major = 0; + + if (sscanf(ver, "%d.", &major) < 1) + die(_("could not parse Spock version \"%s\""), ver); + if (major < 6) + die(_("Spock version %s on source is too old for bidirectional " + "join; require >= 6.0.0"), ver); + } + PQclear(res); + + /* track_commit_timestamp must be on at the source */ + res = PQexec(source_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check track_commit_timestamp: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + die(_("track_commit_timestamp must be on for bidirectional join (source)")); + PQclear(res); + + /* No pending DDL in spock.queue */ + res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.queue"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not check spock.queue: %s"), + PQerrorMessage(source_conn)); + if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + die(_("pending DDL in spock.queue; wait for replication to drain " + "before joining")); + PQclear(res); + + /* Full-mesh assertion: subscriptions on source == num_peers */ + res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.subscription"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not count subscriptions: %s"), + PQerrorMessage(source_conn)); + { + int sub_count = atoi(PQgetvalue(res, 0, 0)); + + if (sub_count != num_peers) + die(_("source node has %d active subscription(s) but %d peer(s) " + "discovered; partial-mesh topologies are not supported"), + sub_count, num_peers); + } + PQclear(res); + + /* + * Per-peer: connectivity and track_commit_timestamp. + * + * Spock version is not checked on peers here; peer version checking is + * deferred to the subscription-setup phase. + */ + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + + print_msg(VERBOSITY_VERBOSE, + _(" checking peer %s ...\n"), peers[i].node_name); + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + res = PQexec(peer_conn, "SHOW track_commit_timestamp"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + PQfinish(peer_conn); + die(_("could not check track_commit_timestamp on peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "on") != 0) + { + PQclear(res); + PQfinish(peer_conn); + die(_("track_commit_timestamp must be on for bidirectional join " + "(peer \"%s\")"), peers[i].node_name); + } + PQclear(res); + PQfinish(peer_conn); + } + + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); +} + +/* + * Write the bidirectional state manifest to state->manifest_path + * atomically (write to .tmp, then rename). The manifest is a simple + * hand-formatted JSON file, with string values escaped by + * append_json_string(). + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + char tmp_path[MAXPGPATH]; + FILE *f; + int i; + + snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); + + appendPQExpBufferStr(buf, "{\n"); + appendPQExpBufferStr(buf, " \"version\": 1,\n"); + + appendPQExpBufferStr(buf, " \"subscriber_name\": \""); + append_json_string(buf, subscriber_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"dbname\": \""); + append_json_string(buf, dbname); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_dsn\": \""); + append_json_string(buf, source_dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_slot_name\": \""); + if (state->source_slot_name) + append_json_string(buf, state->source_slot_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_origin_name\": \""); + if (state->source_origin_name) + append_json_string(buf, state->source_origin_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peers\": [\n"); + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *p = &state->peers[i]; + bool last = (i == state->num_peers - 1); + + appendPQExpBufferStr(buf, " {\n"); + + appendPQExpBufferStr(buf, " \"node_name\": \""); + append_json_string(buf, p->node_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peer_dsn\": \""); + append_json_string(buf, p->dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"sub_name_on_n3\": \""); + append_json_string(buf, p->sub_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peer_slot_name\": \""); + append_json_string(buf, p->slot_name); + appendPQExpBufferStr(buf, "\"\n"); + + appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); + } + appendPQExpBufferStr(buf, " ]\n"); + appendPQExpBufferStr(buf, "}\n"); + + f = fopen(tmp_path, "w"); + if (f == NULL) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + if (fwrite(buf->data, 1, buf->len, f) != buf->len) + { + fclose(f); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (fclose(f) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, state->manifest_path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + state->manifest_path, strerror(errno)); + + destroyPQExpBuffer(buf); +} + +/* + * Semantic-action state for read_manifest(). Passed as void *semstate to all + * pg_parse_json callbacks; tracks nesting depth and accumulates field values. + */ +typedef struct ManifestParseState +{ + /* outputs written by scalar callback */ + char **subscriber_name_out; + char **dbname_out; + char **source_dsn_out; + BidirectionalState *bidir; + + /* parser context */ + int depth; /* object/array nesting depth */ + bool in_peers; /* inside the top-level "peers" array */ + bool in_peer_obj; /* inside one peer object */ + char *cur_field; /* current object field name (owned by us) */ + + /* per-peer accumulator, flushed on each object_end inside peers */ + char *peer_node_name; + char *peer_dsn; + char *peer_sub_name; + char *peer_slot_name; + int peer_capacity; +} ManifestParseState; + +static JsonParseErrorType +manifest_object_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->in_peers && s->depth == 3) + s->in_peer_obj = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_object_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peer_obj && s->depth == 3) + { + int i = s->bidir->num_peers; + + if (i >= s->peer_capacity) + { + s->peer_capacity = (s->peer_capacity > 0) ? s->peer_capacity * 2 : 4; + s->bidir->peers = pg_realloc(s->bidir->peers, + s->peer_capacity * sizeof(PeerNodeInfo)); + } + s->bidir->peers[i].node_name = s->peer_node_name; + s->bidir->peers[i].dsn = s->peer_dsn; + s->bidir->peers[i].sub_name = s->peer_sub_name; + s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->num_peers++; + s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->in_peer_obj = false; + } + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_start(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + s->depth++; + if (s->depth == 2 && s->cur_field != NULL && + strcmp(s->cur_field, "peers") == 0) + s->in_peers = true; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_array_end(void *st) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->in_peers && s->depth == 2) + s->in_peers = false; + s->depth--; + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_ofield_start(void *st, char *fname, bool isnull) +{ + ManifestParseState *s = (ManifestParseState *) st; + + (void) isnull; + pg_free(s->cur_field); + s->cur_field = pg_strdup(fname); + pg_free(fname); /* callback owns the token */ + return JSON_SUCCESS; +} + +static JsonParseErrorType +manifest_scalar(void *st, char *token, JsonTokenType tokentype) +{ + ManifestParseState *s = (ManifestParseState *) st; + + if (s->cur_field == NULL || tokentype != JSON_TOKEN_STRING) + { + pg_free(token); + return JSON_SUCCESS; + } + + if (!s->in_peer_obj) + { + /* top-level scalar fields */ + if (strcmp(s->cur_field, "subscriber_name") == 0) + *s->subscriber_name_out = token; + else if (strcmp(s->cur_field, "dbname") == 0) + *s->dbname_out = token; + else if (strcmp(s->cur_field, "source_dsn") == 0) + *s->source_dsn_out = token; + else if (strcmp(s->cur_field, "source_slot_name") == 0) + s->bidir->source_slot_name = token; + else if (strcmp(s->cur_field, "source_origin_name") == 0) + s->bidir->source_origin_name = token; + else + pg_free(token); + } + else + { + /* per-peer scalar fields */ + if (strcmp(s->cur_field, "node_name") == 0) + s->peer_node_name = token; + else if (strcmp(s->cur_field, "peer_dsn") == 0) + s->peer_dsn = token; + else if (strcmp(s->cur_field, "sub_name_on_n3") == 0) + s->peer_sub_name = token; + else if (strcmp(s->cur_field, "peer_slot_name") == 0) + s->peer_slot_name = token; + else + pg_free(token); + } + return JSON_SUCCESS; +} + +/* + * Read the bidirectional manifest from manifest_path. Returns false if + * the file does not exist (nothing to clean up); dies if it exists but + * cannot be read or is malformed. On success, sets *subscriber_name_out, + * *dbname_out, *source_dsn_out, and populates state->peers[]. + * + * Uses pg_parse_json (common/jsonapi.h) for JSON lexing, so string + * quoting, escape sequences, and nesting are handled correctly. + */ +static bool +read_manifest(const char *manifest_path, BidirectionalState *state, + char **subscriber_name_out, char **dbname_out, + char **source_dsn_out) +{ + struct stat st; + char *content; + FILE *f; + JsonLexContext *lex; + JsonSemAction sem; + ManifestParseState pstate; + JsonParseErrorType result; + + if (stat(manifest_path, &st) != 0) + return false; + + content = pg_malloc(st.st_size + 1); + f = fopen(manifest_path, "r"); + if (f == NULL) + die(_("could not open manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + + if ((size_t) fread(content, 1, st.st_size, f) != (size_t) st.st_size) + { + fclose(f); + die(_("could not read manifest file \"%s\": %s"), + manifest_path, strerror(errno)); + } + content[st.st_size] = '\0'; + fclose(f); + + memset(&pstate, 0, sizeof(pstate)); + pstate.subscriber_name_out = subscriber_name_out; + pstate.dbname_out = dbname_out; + pstate.source_dsn_out = source_dsn_out; + pstate.bidir = state; + + memset(&sem, 0, sizeof(sem)); + sem.semstate = &pstate; + sem.object_start = manifest_object_start; + sem.object_end = manifest_object_end; + sem.array_start = manifest_array_start; + sem.array_end = manifest_array_end; + sem.object_field_start = manifest_ofield_start; + sem.scalar = manifest_scalar; + + lex = makeJsonLexContextCstringLen(NULL, content, st.st_size, + PG_UTF8, true); + result = pg_parse_json(lex, &sem); + pg_free(content); + pg_free(pstate.cur_field); + + if (result != JSON_SUCCESS) + { + char *detail = json_errdetail(result, lex); + + freeJsonLexContext(lex); + die(_("manifest file \"%s\" is malformed: %s"), manifest_path, detail); + } + freeJsonLexContext(lex); + + if (!*subscriber_name_out || !*dbname_out || !*source_dsn_out) + die(_("manifest file \"%s\" is malformed or missing required fields"), + manifest_path); + + return true; +} + +/* + * Idempotently remove bidirectional join state from all reachable nodes. + * Connects to the source and each peer, drops replication slots and + * reverse subscriptions created during a previous join attempt. All + * operations are best-effort: connectivity failures are logged as + * warnings rather than being fatal. + */ +static void +cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir) +{ + PGconn *source_conn; + PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + int i; + + print_msg(VERBOSITY_NORMAL, + _("Cleaning up partial bidirectional join state ...\n")); + + source_conn = PQconnectdb(source_dsn); + if (PQstatus(source_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; skipping " + "source-side cleanup: %s\n"), + PQerrorMessage(source_conn)); + PQfinish(source_conn); + source_conn = NULL; + } + + /* Drop source replication slot if it was created */ + if (source_conn && state->source_slot_name && state->source_slot_name[0]) + { + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + state->source_slot_name); + res = PQexec(source_conn, query->data); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped source slot %s\n"), + state->source_slot_name); + PQclear(res); + } + + /* Per-peer: drop slot and any reverse subscription */ + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + char reverse_sub[NAMEDATALEN]; + + if (!peer->dsn || !peer->dsn[0]) + continue; + + peer_conn = PQconnectdb(peer->dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to peer \"%s\"; skipping " + "peer-side cleanup: %s\n"), + peer->node_name, PQerrorMessage(peer_conn)); + PQfinish(peer_conn); + continue; + } + + if (peer->slot_name && peer->slot_name[0]) + { + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + peer->slot_name); + res = PQexec(peer_conn, query->data); + if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped peer slot %s on %s\n"), + peer->slot_name, peer->node_name); + PQclear(res); + } + + /* + * Drop the reverse subscription (peer -> new subscriber) if it was + * created during a previous attempt. The sub_drop second argument + * is ifexists=true. + */ + snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", + peer->node_name, subscriber_name); + printfPQExpBuffer(query, + "SELECT spock.sub_drop('%s', true)", + reverse_sub); + res = PQexec(peer_conn, query->data); + PQclear(res); + + PQfinish(peer_conn); + print_msg(VERBOSITY_NORMAL, + _(" cleaned up peer %s\n"), peer->node_name); + } + + if (source_conn) + PQfinish(source_conn); + + destroyPQExpBuffer(query); + + if (state->manifest_path && state->manifest_path[0]) + { + unlink(state->manifest_path); + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + } + + print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); +} + int main(int argc, char **argv) @@ -194,6 +882,8 @@ main(int argc, char **argv) logfd; char *restore_point_name = NULL; char *extra_basebackup_args = NULL; + BidirectionalState bidir = {0}; + char bidir_manifest_path[MAXPGPATH] = {0}; static struct option long_options[] = { {"subscriber-name", required_argument, NULL, 'n'}, @@ -210,6 +900,10 @@ main(int argc, char **argv) {"databases", required_argument, NULL, 9}, {"extra-basebackup-args", required_argument, NULL, 10}, {"text-types", no_argument, NULL, 11}, + {"bidirectional", no_argument, NULL, 12}, + {"stall-timeout", required_argument, NULL, 13}, + {"max-wait", required_argument, NULL, 14}, + {"cleanup", no_argument, NULL, 15}, {NULL, 0, NULL, 0} }; @@ -299,6 +993,22 @@ main(int argc, char **argv) case 11: force_text_transfer = true; break; + case 12: + bidir.enabled = true; + break; + case 13: + bidir.stall_timeout = atoi(optarg); + if (bidir.stall_timeout <= 0) + die(_("--stall-timeout must be a positive integer")); + break; + case 14: + bidir.max_wait = atoi(optarg); + if (bidir.max_wait < 0) + die(_("--max-wait must be a non-negative integer")); + break; + case 15: + bidir.cleanup_mode = true; + break; default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -316,16 +1026,20 @@ main(int argc, char **argv) fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - else if (subscriber_name == NULL) + else if (subscriber_name == NULL && !bidir.cleanup_mode) { fprintf(stderr, _("No subscriber name specified\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - if (!base_prov_connstr || !strlen(base_prov_connstr)) + if (bidir.cleanup_mode && !bidir.enabled) + die(_("--cleanup requires --bidirectional.\n")); + + if (!bidir.cleanup_mode && (!base_prov_connstr || !strlen(base_prov_connstr))) die(_("Provider connection string must be specified.\n")); - if (!base_sub_connstr || !strlen(base_sub_connstr)) + if (!bidir.enabled && !bidir.cleanup_mode && + (!base_sub_connstr || !strlen(base_sub_connstr))) die(_("Subscriber connection string must be specified.\n")); if (apply_delay < 0) @@ -337,6 +1051,33 @@ main(int argc, char **argv) if (!replication_sets || !strlen(replication_sets)) replication_sets = "default,default_insert_only,ddl_sql"; + /* Build the manifest path from --pgdata */ + if (bidir.enabled || bidir.cleanup_mode) + { + snprintf(bidir_manifest_path, MAXPGPATH, + "%s/spock_bidirectional_manifest.json", data_dir); + bidir.manifest_path = bidir_manifest_path; + if (bidir.stall_timeout == 0) + bidir.stall_timeout = 600; + } + + /* --cleanup: read manifest, remove partial state, exit */ + if (bidir.cleanup_mode) + { + char *sub_name = NULL; + char *db = NULL; + char *src_dsn = NULL; + + if (!read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) + { + fprintf(stderr, _("No manifest found at %s; nothing to clean up.\n"), + bidir.manifest_path); + exit(0); + } + cleanup_partial_state(&bidir, sub_name, db, src_dsn, false); + exit(0); + } + /* Init random numbers used for slot suffixes, etc */ srand(time(NULL)); @@ -372,9 +1113,12 @@ main(int argc, char **argv) if (!prov_connstr || !strlen(prov_connstr)) die(_("Provider connection string is not valid.\n")); - sub_connstr = get_connstr(base_sub_connstr, db); - if (!sub_connstr || !strlen(sub_connstr)) - die(_("Subscriber connection string is not valid.\n")); + if (!bidir.enabled) + { + sub_connstr = get_connstr(base_sub_connstr, db); + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + } } /* @@ -408,6 +1152,29 @@ main(int argc, char **argv) provider_conn = connectdb(prov_connstr); remote_info = get_remote_info(provider_conn); + /* + * --bidirectional: discover peers, verify preconditions, write the + * manifest, then exit. The rest of the join resumes from this + * manifest once the physical backup has been taken and the + * subscriber is running. + */ + if (bidir.enabled) + { + bidir.num_peers = discover_peer_nodes(provider_conn, + remote_info->node_name, + subscriber_name, db, + &bidir.peers); + check_preconditions(provider_conn, bidir.peers, bidir.num_peers); + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "preconditions OK, manifest written to %s.\n"), + bidir.num_peers, bidir.manifest_path); + PQfinish(provider_conn); + provider_conn = NULL; + exit(0); + } + /* only need to do this piece once */ if (dbnum == 0) From 3c0c6b595605176cb947e9fbc3194015cf61a618 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 13 Jul 2026 19:00:47 +0500 Subject: [PATCH 06/14] spock_create_subscriber: add TAP test for --bidirectional plumbing --- tests/tap/schedule | 2 + tests/tap/t/047_bidir_plumbing.pl | 165 ++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 tests/tap/t/047_bidir_plumbing.pl diff --git a/tests/tap/schedule b/tests/tap/schedule index 7e8c457ea..4d8b2943c 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,6 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts +test: 047_bidir_plumbing # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # @@ -65,3 +66,4 @@ test: 045_lsn_from_commit_ts # Regression tests test: 103_manager_worker_dboid_race test: 105_sub_disable_retransmit_after_disconnect + diff --git a/tests/tap/t/047_bidir_plumbing.pl b/tests/tap/t/047_bidir_plumbing.pl new file mode 100644 index 000000000..64d19f12f --- /dev/null +++ b/tests/tap/t/047_bidir_plumbing.pl @@ -0,0 +1,165 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 047_bidir_plumbing.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the plumbing phase of the SPOC-601 bidirectional node-join +# procedure. The test does NOT start a third PostgreSQL instance; it only +# exercises the utility's plumbing phase against an existing 2-node cluster: +# +# --bidirectional discover peers, check preconditions, write manifest +# --cleanup idempotently remove partial state / manifest +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, track_commit_timestamp=on) +# +# The utility is run with --pgdata pointing at a plain temp directory (no PG +# cluster) that exists solely to hold the manifest file. +# +# Test count breakdown: +# 1 binary found +# 1 temp pgdata created +# 5 create_cluster(2) [2 pg_isready + 2 spock checks + 1 pass] +# 1 cross_wire n1<->n2 +# 1 --bidirectional exits 0 +# 1 manifest file written +# 1 manifest: version 1 +# 1 manifest: subscriber_name n3 +# 1 manifest: dbname regression +# 1 manifest: source_dsn present +# 1 manifest: peer n2 listed +# 1 manifest: peer_slot_name present +# 1 --cleanup exits 0 +# 1 manifest removed +# 1 --cleanup with no manifest exits 0 (idempotent) +# 1 destroy_cluster +# --- +# 20 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 20; +use File::Path qw(remove_tree make_path); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query psql_or_bail); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + # Fall back to the build tree (CWD is tests/tap/ during make check_prove) + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# Scratch directory that stands in for n3's future PGDATA. +# It just needs to exist so the manifest can be written there. +# ============================================================================= +my $N3_PGDATA = '/tmp/spock_bidir_test_n3_pgdata'; +my $MANIFEST = "$N3_PGDATA/spock_bidirectional_manifest.json"; + +remove_tree($N3_PGDATA) if -d $N3_PGDATA; +make_path($N3_PGDATA) + or BAIL_OUT("could not create temp pgdata dir: $N3_PGDATA"); +pass("temp pgdata dir for n3 created"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# create_cluster counts as 5 tests (pg_isready + spock check per node + pass) +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +# Create bidirectional subscriptions n1->n2 and n2->n1 (1 test) +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# TEST: --bidirectional mode +# Discovers peer n2 from n1, checks preconditions, writes manifest. +# ============================================================================= + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $N3_PGDATA, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + ], + '--bidirectional plumbing exits 0' +); + +ok(-f $MANIFEST, + 'manifest written to /spock_bidirectional_manifest.json'); + +# Read and inspect manifest content +my $manifest_content = ''; +if (-f $MANIFEST) { + open my $fh, '<', $MANIFEST or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +like($manifest_content, qr/"version":\s*1/, + 'manifest: version is 1'); +like($manifest_content, qr/"subscriber_name":\s*"n3"/, + 'manifest: subscriber_name is n3'); +like($manifest_content, qr/"dbname":\s*"$dbname"/, + 'manifest: dbname matches provider dbname'); +ok(index($manifest_content, '"source_dsn":') >= 0, + 'manifest: source_dsn field present'); +like($manifest_content, qr/"node_name":\s*"n2"/, + 'manifest: peer n2 is listed in peers array'); +ok(index($manifest_content, '"peer_slot_name":') >= 0, + 'manifest: peer_slot_name field present'); + +# ============================================================================= +# TEST: --cleanup mode — removes manifest, exits 0 +# ============================================================================= + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--pgdata', $N3_PGDATA, + ], + '--cleanup with manifest exits 0' +); + +ok(!-f $MANIFEST, + 'manifest file removed by --cleanup'); + +# Second cleanup with no manifest must also exit 0 (idempotent) +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--pgdata', $N3_PGDATA, + ], + '--cleanup with no manifest exits 0 (idempotent)' +); + +# ============================================================================= +# CLEANUP +# ============================================================================= +remove_tree($N3_PGDATA); +destroy_cluster('Cleanup'); From ee93acf118399b933778cb1f7d53f99246c67ce9 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Tue, 28 Jul 2026 14:01:41 +0500 Subject: [PATCH 07/14] spock_create_subscriber: physical backup, catalog strip, repset restore Continue --bidirectional past peer discovery into the physical-backup pipeline (source slot creation, pg_basebackup, restore point, recovery), stopping just before the catchup subscription: - Capture replication-set definitions, table memberships, and sequence state from the local catalog before DROP EXTENSION removes it. - Drop all replication origins, then guard DROP EXTENSION ... CASCADE with a pg_depend inventory of non-spock dependents, failing loudly instead of silently destroying user objects. - Create the local node, immediately set spock.readonly = 'local', and restore the captured replication-set state. The DSN registered for the node is derived from --subscriber-dsn, not a separate option. - Give the new node its own system identifier right after promotion and before any catalog mutation, so a plain system_identifier comparison can verify --subscriber-dsn actually reaches it. - New --force option: also remove the data directory on --cleanup. Preconditions are hardened beyond a simple subscription count: full-mesh validation now checks actual subscription health and peer identity, not just sub_enabled, and rejects duplicate edges from the same origin; replication-set and schema fingerprints are compared between the source and every peer, covering column type, typmod, collation, nullability, generated/identity columns, relation kind, and replica identity. --- .gitignore | 2 + src/spock_fe.c | 9 +- tests/tap/schedule | 2 +- tests/tap/t/047_bidir_plumbing.pl | 165 - tests/tap/t/048_bidir_pr3.pl | 585 ++++ .../spock_create_subscriber.c | 2682 +++++++++++++++-- 6 files changed, 2978 insertions(+), 467 deletions(-) delete mode 100644 tests/tap/t/047_bidir_plumbing.pl create mode 100644 tests/tap/t/048_bidir_pr3.pl diff --git a/.gitignore b/.gitignore index cc0c7c0a4..a798543d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ results regression_output tmp_check +/utils/spock_create_subscriber/spock_create_subscriber +/utils/spock_create_subscriber/.deps/ .vimrc *.o *.so diff --git a/src/spock_fe.c b/src/spock_fe.c index ebe62a825..0a6d66e04 100644 --- a/src/spock_fe.c +++ b/src/spock_fe.c @@ -226,10 +226,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -238,6 +240,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) diff --git a/tests/tap/schedule b/tests/tap/schedule index 4d8b2943c..3502d4916 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,7 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts -test: 047_bidir_plumbing +test: 048_bidir_pr3 # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/047_bidir_plumbing.pl b/tests/tap/t/047_bidir_plumbing.pl deleted file mode 100644 index 64d19f12f..000000000 --- a/tests/tap/t/047_bidir_plumbing.pl +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/perl -# ============================================================================= -# Test: 047_bidir_plumbing.pl - spock_create_subscriber --bidirectional -# ============================================================================= -# Validates the plumbing phase of the SPOC-601 bidirectional node-join -# procedure. The test does NOT start a third PostgreSQL instance; it only -# exercises the utility's plumbing phase against an existing 2-node cluster: -# -# --bidirectional discover peers, check preconditions, write manifest -# --cleanup idempotently remove partial state / manifest -# -# Topology: -# n1 <-> n2 (full bidirectional Spock subscriptions, track_commit_timestamp=on) -# -# The utility is run with --pgdata pointing at a plain temp directory (no PG -# cluster) that exists solely to hold the manifest file. -# -# Test count breakdown: -# 1 binary found -# 1 temp pgdata created -# 5 create_cluster(2) [2 pg_isready + 2 spock checks + 1 pass] -# 1 cross_wire n1<->n2 -# 1 --bidirectional exits 0 -# 1 manifest file written -# 1 manifest: version 1 -# 1 manifest: subscriber_name n3 -# 1 manifest: dbname regression -# 1 manifest: source_dsn present -# 1 manifest: peer n2 listed -# 1 manifest: peer_slot_name present -# 1 --cleanup exits 0 -# 1 manifest removed -# 1 --cleanup with no manifest exits 0 (idempotent) -# 1 destroy_cluster -# --- -# 20 total -# ============================================================================= - -use strict; -use warnings; -use Test::More tests => 20; -use File::Path qw(remove_tree make_path); -use lib '.'; -use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail - command_ok system_maybe get_test_config scalar_query psql_or_bail); - -# ============================================================================= -# Locate spock_create_subscriber binary -# ============================================================================= -my $SCS_BIN; -for my $dir (split(':', $ENV{PATH} // '')) { - my $c = "$dir/spock_create_subscriber"; - if (-x $c) { $SCS_BIN = $c; last; } -} -unless (defined $SCS_BIN) { - # Fall back to the build tree (CWD is tests/tap/ during make check_prove) - my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; - $SCS_BIN = $bt if -x $bt; -} -BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") - unless defined $SCS_BIN; -pass("spock_create_subscriber binary found"); - -# ============================================================================= -# Scratch directory that stands in for n3's future PGDATA. -# It just needs to exist so the manifest can be written there. -# ============================================================================= -my $N3_PGDATA = '/tmp/spock_bidir_test_n3_pgdata'; -my $MANIFEST = "$N3_PGDATA/spock_bidirectional_manifest.json"; - -remove_tree($N3_PGDATA) if -d $N3_PGDATA; -make_path($N3_PGDATA) - or BAIL_OUT("could not create temp pgdata dir: $N3_PGDATA"); -pass("temp pgdata dir for n3 created"); - -# ============================================================================= -# SETUP: 2-node cluster, cross-wired bidirectionally -# create_cluster counts as 5 tests (pg_isready + spock check per node + pass) -# ============================================================================= -create_cluster(2, 'Create bidirectional 2-node cluster'); - -my $config = get_test_config(); -my $node_ports = $config->{node_ports}; -my $dbname = $config->{db_name}; -my $host = $config->{host}; -my $db_user = $config->{db_user}; -my $db_password = $config->{db_password}; - -my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" - . " user=$db_user password=$db_password"; - -# Create bidirectional subscriptions n1->n2 and n2->n1 (1 test) -cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); - -# ============================================================================= -# TEST: --bidirectional mode -# Discovers peer n2 from n1, checks preconditions, writes manifest. -# ============================================================================= - -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--pgdata', $N3_PGDATA, - '--subscriber-name', 'n3', - '--provider-dsn', $n1_dsn, - ], - '--bidirectional plumbing exits 0' -); - -ok(-f $MANIFEST, - 'manifest written to /spock_bidirectional_manifest.json'); - -# Read and inspect manifest content -my $manifest_content = ''; -if (-f $MANIFEST) { - open my $fh, '<', $MANIFEST or die "Cannot read manifest: $!"; - local $/; - $manifest_content = <$fh>; - close $fh; -} - -like($manifest_content, qr/"version":\s*1/, - 'manifest: version is 1'); -like($manifest_content, qr/"subscriber_name":\s*"n3"/, - 'manifest: subscriber_name is n3'); -like($manifest_content, qr/"dbname":\s*"$dbname"/, - 'manifest: dbname matches provider dbname'); -ok(index($manifest_content, '"source_dsn":') >= 0, - 'manifest: source_dsn field present'); -like($manifest_content, qr/"node_name":\s*"n2"/, - 'manifest: peer n2 is listed in peers array'); -ok(index($manifest_content, '"peer_slot_name":') >= 0, - 'manifest: peer_slot_name field present'); - -# ============================================================================= -# TEST: --cleanup mode — removes manifest, exits 0 -# ============================================================================= - -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--cleanup', - '--pgdata', $N3_PGDATA, - ], - '--cleanup with manifest exits 0' -); - -ok(!-f $MANIFEST, - 'manifest file removed by --cleanup'); - -# Second cleanup with no manifest must also exit 0 (idempotent) -command_ok( - [ $SCS_BIN, - '--bidirectional', - '--cleanup', - '--pgdata', $N3_PGDATA, - ], - '--cleanup with no manifest exits 0 (idempotent)' -); - -# ============================================================================= -# CLEANUP -# ============================================================================= -remove_tree($N3_PGDATA); -destroy_cluster('Cleanup'); diff --git a/tests/tap/t/048_bidir_pr3.pl b/tests/tap/t/048_bidir_pr3.pl new file mode 100644 index 000000000..a317e4515 --- /dev/null +++ b/tests/tap/t/048_bidir_pr3.pl @@ -0,0 +1,585 @@ +#!/usr/bin/perl +# ============================================================================= +# Test: 048_bidir_pr3.pl - spock_create_subscriber --bidirectional +# ============================================================================= +# Validates the bidirectional node-join procedure: physical backup, recovery +# to a restore point, catalog strip (capture + origin drop + guarded DROP +# EXTENSION), and replication-set/table/sequence restore -- stopping short +# of the catchup subscription (a later step). +# +# Topology: +# n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node +# cluster from create_cluster/cross_wire) +# n3 a real third PostgreSQL instance built via +# `spock_create_subscriber --bidirectional`, physically backed +# up from n1. +# +# Test count breakdown: +# 1 binary found +# 5 create_cluster(2) +# 1 cross_wire n1<->n2 +# 1 custom replication set created on n1 +# 1 table with row_filter added to custom set on n1 +# 1 table with explicit column list added to custom set on n1 +# 1 sequence added to custom set on n1 +# 1 sequence advanced past its initial value on n1 (setval fidelity check) +# 1 partitioned table (parent + 2 children) added to custom set on n1 +# 1 sequence with apostrophe in name added to custom set on n1 +# 1 --bidirectional exits 0 +# 1 n3 postgres is running +# 1 spock extension installed cleanly on n3 (exactly one row) +# 1 n3 has no leftover replication origins from the basebackup +# 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 +# 1 spock.readonly is 'local' on n3 +# 1 custom replication set restored on n3 with correct flags +# 1 table membership restored with correct row_filter +# 1 table membership restored with correct explicit column list +# 1 sequence value restored exactly (last_value) +# 1 sequence is_called restored exactly +# 1 sequence pr3_test_seq is a member of pr3_test_repset on n3 +# 1 partitioned table parent + 2 children all present in repset on n3 +# 1 apostrophe-named sequence value restored on n3 +# 1 apostrophe-named sequence is_called restored on n3 +# 1 apostrophe-named sequence is a member of pr3_test_repset on n3 +# 1 manifest: source_slot_name populated +# 1 manifest: source_restore_lsn populated +# 1 manifest: node_dsn populated +# 1 source slot exists on n1 +# 1 --cleanup --force exits 0 +# 1 source slot removed from n1 after cleanup +# 1 n3 data directory removed after cleanup --force +# 1 manifest removed after cleanup +# 1 --bidirectional rejects a multi-database request +# 1 --bidirectional aborts when another database on the source has spock configured +# 1 --bidirectional rejects a broken full-mesh topology (disabled subscription) +# 1 --bidirectional rejects mismatched replication-set flags between source and peer +# 1 a broken --extra-basebackup-args makes the base backup fail +# 1 pending-cleanup sidecar written before the failed backup +# 1 pending-cleanup sidecar is mode 0600 +# 1 source slot still exists on n1 after the failed backup (orphaned) +# 1 --cleanup --force recovers via the pending sidecar +# 1 source slot removed from n1 via sidecar-based cleanup +# 1 pending-cleanup sidecar removed after cleanup +# 1 a broken backup orphans a slot for the retry-cleanup test +# 1 pending sidecar written for the retry-cleanup test +# 1 --cleanup exits non-zero when the source is unreachable +# 1 pending sidecar retained after an incomplete cleanup +# 1 n1 postgres is running again +# 1 --cleanup --force succeeds once the source is reachable again +# 1 pending sidecar removed once cleanup actually completed +# 1 destroy_cluster +# --- +# 57 total +# ============================================================================= + +use strict; +use warnings; +use Test::More tests => 57; +use File::Path qw(remove_tree); +use lib '.'; +use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail + command_ok system_maybe get_test_config scalar_query + psql_or_bail wait_for_pg_ready); + +# ============================================================================= +# Locate spock_create_subscriber binary +# ============================================================================= +my $SCS_BIN; +for my $dir (split(':', $ENV{PATH} // '')) { + my $c = "$dir/spock_create_subscriber"; + if (-x $c) { $SCS_BIN = $c; last; } +} +unless (defined $SCS_BIN) { + my $bt = '../../utils/spock_create_subscriber/spock_create_subscriber'; + $SCS_BIN = $bt if -x $bt; +} +BAIL_OUT("spock_create_subscriber binary not found; run 'make install' first") + unless defined $SCS_BIN; +pass("spock_create_subscriber binary found"); + +# ============================================================================= +# SETUP: 2-node cluster, cross-wired bidirectionally +# ============================================================================= +create_cluster(2, 'Create bidirectional 2-node cluster'); + +my $config = get_test_config(); +my $node_ports = $config->{node_ports}; +my $dbname = $config->{db_name}; +my $host = $config->{host}; +my $db_user = $config->{db_user}; +my $db_password = $config->{db_password}; +my $pg_bin = $config->{pg_bin}; + +my $n1_dsn = "host=$host port=$node_ports->[0] dbname=$dbname" + . " user=$db_user password=$db_password"; + +my $n1_sysid = scalar_query(1, "SELECT system_identifier FROM pg_control_system()"); + +cross_wire(2, ['n1', 'n2'], 'Cross-wire n1 <-> n2 bidirectionally'); + +# ============================================================================= +# Seed n1 with a custom replication set, a table with a row_filter, and a +# sequence, to exercise the catalog capture/restore with non-default state +# rather than just the three built-in sets. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_create('pr3_test_repset', true, true, true, false)"; +pass('custom replication set created on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_tbl (id serial primary key, region text, value integer)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_tbl', synchronize_data := false, " . + "row_filter := 'region = ''east''')"; +pass('table with row_filter added to custom set on n1'); + +# Table with an explicit, non-default column list, to exercise the +# columns := restore path (captured/restored as a bare array- +# literal string relying on implicit text[] coercion) -- previously +# untested, so a round-trip regression here could pass silently. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_cols (id serial primary key, region text, " . + "value integer, secret text)"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_cols', synchronize_data := false, " . + "columns := ARRAY['id', 'region', 'value'])"; +pass('table with explicit column list added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE SEQUENCE pr3_test_seq"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_seq('pr3_test_repset', 'pr3_test_seq')"; +pass('sequence added to custom set on n1'); + +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT setval('pr3_test_seq', 42, true)"; +my $seq_before = scalar_query(1, "SELECT last_value FROM pr3_test_seq"); +is($seq_before, '42', 'sequence advanced past its initial value on n1'); + +# Partitioned table: parent + 2 children get separate captured membership +# rows (that's how include_partitions => true populated them here); restore +# must not try to re-add children a second time via the parent's own +# include_partitions => true call. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part (id int, region text, PRIMARY KEY (id, region)) PARTITION BY LIST (region)"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_east PARTITION OF pr3_test_part FOR VALUES IN ('east')"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr3_test_part_west PARTITION OF pr3_test_part FOR VALUES IN ('west')"; +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_add_table(set_name := 'pr3_test_repset', " . + "relation := 'pr3_test_part', synchronize_data := false, " . + "include_partitions := true)"; +pass('partitioned table (parent + 2 children) added to custom set on n1'); + +# Sequence with an apostrophe in its name, to exercise setval() quoting. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(CREATE SEQUENCE "weird's_seq"); +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT spock.repset_add_seq('pr3_test_repset', '"weird''s_seq"')); +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + q(SELECT setval('"weird''s_seq"', 7, true)); +pass('sequence with apostrophe in name added to custom set on n1'); + +# check_preconditions() requires all of n1's outbound replication to have +# caught up (no unreplicated DDL/data still in flight to n2); wait for the +# setup above to drain. +for (1 .. 15) { + my $lag = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots" . + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" . + " AND (confirmed_flush_lsn IS NULL OR confirmed_flush_lsn < pg_current_wal_lsn())"); + last if defined $lag && $lag eq '0'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional continues through physical backup / catalog strip / +# repset restore, stopping before the catchup subscription. +# ============================================================================= +my $n3_port = $node_ports->[1] + 1; +my $n3_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3'; +my $manifest = "$n3_datadir/spock_bidirectional_manifest.json"; +my $n3_dsn = "host=$host port=$n3_port dbname=$dbname" + . " user=$db_user password=$db_password"; + +remove_tree($n3_datadir) if -d $n3_datadir; + +# n3's postgresql.conf is copied verbatim from n1 by the basebackup, port and +# all -- since all nodes run on the same host in this test, n3 must be given +# an override with its own port (a real cross-host join wouldn't need this). +my $n3_conf = '/tmp/tmp_spock_node_2_postgresql.conf.override'; +open my $conf_fh, '>', $n3_conf or die "Cannot write $n3_conf: $!"; +print $conf_fh "shared_buffers=1GB\n"; +print $conf_fh "shared_preload_libraries='spock'\n"; +print $conf_fh "wal_level=logical\n"; +print $conf_fh "spock.enable_ddl_replication=on\n"; +print $conf_fh "spock.include_ddl_repset=on\n"; +print $conf_fh "spock.allow_ddl_from_functions=on\n"; +print $conf_fh "spock.exception_behaviour=sub_disable\n"; +print $conf_fh "spock.conflict_resolution=last_update_wins\n"; +print $conf_fh "track_commit_timestamp=on\n"; +print $conf_fh "spock.exception_replay_queue_size='1MB'\n"; +print $conf_fh "spock.enable_spill=on\n"; +print $conf_fh "port=$n3_port\n"; +print $conf_fh "listen_addresses='*'\n"; +print $conf_fh "logging_collector=on\n"; +print $conf_fh "log_directory='" . $config->{log_dir} . "'\n"; +print $conf_fh "log_filename='00${n3_port}.log'\n"; +close $conf_fh; + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--pgdata', $n3_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--postgresql-conf', $n3_conf, + ], + '--bidirectional exits 0' +); + +ok(wait_for_pg_ready($host, $n3_port, $pg_bin, 30), 'n3 postgres is running'); + +my $ext_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_extension WHERE extname = 'spock'"`; +$ext_count =~ s/\s+//g; +is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); + +my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; +$origin_count =~ s/\s+//g; +is($origin_count, '0', 'n3 has no leftover replication origins from the basebackup'); + +my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; +$n3_sysid =~ s/\s+//g; +isnt($n3_sysid, $n1_sysid, + 'n3 was given its own system identifier (pg_resetwal), distinct from n1'); + +my $readonly = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SHOW spock.readonly"`; +$readonly =~ s/\s+//g; +is($readonly, 'local', "spock.readonly is 'local' on n3"); + +my $repset_flags = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT replicate_insert, replicate_update, replicate_delete, replicate_truncate FROM spock.replication_set WHERE set_name = 'pr3_test_repset'"`; +$repset_flags =~ s/\s+//g; +is($repset_flags, 't|t|t|f', 'custom replication set restored on n3 with correct flags'); + +my $row_filter = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT pg_get_expr(rts.set_row_filter, rts.set_reloid) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset'"`; +$row_filter =~ s/^\s+|\s+$//g; +like($row_filter, qr/region\s*=\s*'east'/, 'table membership restored with correct row_filter'); + +my $columns = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT rts.set_att_list FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text = 'pr3_test_cols'"`; +$columns =~ s/^\s+|\s+$//g; +is($columns, '{id,region,value}', + 'table membership restored with correct explicit column list'); + +my $seq_last_value = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT last_value FROM pr3_test_seq"`; +$seq_last_value =~ s/\s+//g; +is($seq_last_value, '42', 'sequence value restored exactly (last_value)'); + +my $seq_is_called = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT is_called FROM pr3_test_seq"`; +$seq_is_called =~ s/\s+//g; +is($seq_is_called, 't', 'sequence is_called restored exactly'); + +# pr3_test_seq must be an actual member of pr3_test_repset on n3, not just +# have its value restored (a regression here is the sequence-membership bug: +# setval() alone leaves the sequence unpublished). +my $seq_member = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = 'pr3_test_seq'"`; +$seq_member =~ s/\s+//g; +is($seq_member, '1', 'sequence pr3_test_seq is a member of pr3_test_repset on n3'); + +# Partitioned table: parent + 2 children must all be present as distinct +# memberships (a regression here is include_partitions => true re-adding +# already-captured children and violating the (set_id, set_reloid) PK, +# which would have aborted the join above rather than just miscounting). +my $part_member_count = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT COUNT(*) FROM spock.replication_set_table rts JOIN spock.replication_set rs ON rts.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rts.set_reloid::regclass::text LIKE 'pr3_test_part%'"`; +$part_member_count =~ s/\s+//g; +is($part_member_count, '3', 'partitioned table parent + 2 children all present in repset on n3'); + +# Sequence with an apostrophe in its name: value/is_called restored and +# membership present, without a SQL syntax error breaking the whole run. +sub psql_capture { + my (@args) = @_; + open(my $fh, '-|', "$pg_bin/psql", @args) or die "cannot run psql: $!"; + local $/; + my $out = <$fh>; + close $fh; + $out =~ s/^\s+|\s+$//g if defined $out; + return $out; +} + +my $weird_seq_value = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT last_value FROM "weird\'s_seq"'); +is($weird_seq_value, '7', "apostrophe-named sequence value restored on n3"); + +my $weird_seq_called = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', 'SELECT is_called FROM "weird\'s_seq"'); +is($weird_seq_called, 't', "apostrophe-named sequence is_called restored on n3"); + +my $weird_seq_member = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM spock.replication_set_seq rss JOIN spock.replication_set rs ON rss.set_id = rs.set_id WHERE rs.set_name = 'pr3_test_repset' AND rss.set_seqoid::regclass::text = '\"weird''s_seq\"'"); +is($weird_seq_member, '1', "apostrophe-named sequence is a member of pr3_test_repset on n3"); + +# ============================================================================= +# Manifest content checks +# ============================================================================= +my $manifest_content = ''; +if (-f $manifest) { + open my $fh, '<', $manifest or die "Cannot read manifest: $!"; + local $/; + $manifest_content = <$fh>; + close $fh; +} + +ok($manifest_content =~ /"source_slot_name":\s*"[^"]+"/, + 'manifest: source_slot_name populated'); +ok($manifest_content =~ /"source_restore_lsn":\s*"[0-9A-Fa-f]+\/[0-9A-Fa-f]+"/, + 'manifest: source_restore_lsn populated'); +ok($manifest_content =~ /"node_dsn":\s*"[^"]+"/, + 'manifest: node_dsn populated'); + +my $source_slot_exists = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($source_slot_exists >= 1, 'source slot exists on n1'); + +# ============================================================================= +# TEST: --cleanup --force removes source slot, data directory, and manifest +# ============================================================================= +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $n3_datadir, + ], + '--cleanup --force exits 0' +); + +my $source_slot_after = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($source_slot_after, '0', 'source slot removed from n1 after cleanup'); + +ok(!-d $n3_datadir, 'n3 data directory removed after cleanup --force'); +ok(!-f $manifest, 'manifest removed after cleanup'); + +# ============================================================================= +# TEST: --bidirectional hard-rejects a multi-database request outright, +# rather than silently joining only the first-named database -- all join +# state is per-database. +# ============================================================================= +my $multidb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_multidb'; +remove_tree($multidb_datadir) if -d $multidb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $multidb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--databases', "$dbname,postgres"), + '--bidirectional rejects a multi-database request'); +remove_tree($multidb_datadir) if -d $multidb_datadir; + +# ============================================================================= +# TEST: --bidirectional aborts if the source instance has spock configured +# on another database too, even though that database was never named via +# --databases (check_single_spock_database() must fail closed). +# ============================================================================= +system_or_bail "$pg_bin/createdb", '-p', $node_ports->[0], 'pr3_other_db'; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "CREATE EXTENSION spock"; +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_create('pr3_other_node', 'dbname=pr3_other_db')"; + +my $otherdb_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_otherdb'; +remove_tree($otherdb_datadir) if -d $otherdb_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $otherdb_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional aborts when another database on the source has spock configured'); +remove_tree($otherdb_datadir) if -d $otherdb_datadir; + +system_maybe "$pg_bin/psql", '-p', $node_ports->[0], '-d', 'pr3_other_db', '-c', + "SELECT spock.node_drop('pr3_other_node')"; +system_maybe "$pg_bin/dropdb", '-p', $node_ports->[0], 'pr3_other_db'; + +# ============================================================================= +# TEST: --bidirectional rejects a broken full-mesh topology -- a disabled +# subscription is not a valid mesh edge, even though it still exists. A +# plain subscription COUNT would not catch this. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_disable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 'f'; + sleep(1); +} + +my $mesh_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_mesh'; +remove_tree($mesh_datadir) if -d $mesh_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $mesh_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects a broken full-mesh topology (disabled subscription)'); +remove_tree($mesh_datadir) if -d $mesh_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SELECT spock.sub_enable('sub_n2_n1', true)"; +for (1 .. 15) { + my $enabled = scalar_query(2, + "SELECT sub_enabled FROM spock.subscription WHERE sub_name = 'sub_n2_n1'"); + last if defined $enabled && $enabled eq 't'; + sleep(1); +} + +# ============================================================================= +# TEST: --bidirectional rejects mismatched replication-set definitions for a +# selected (subscription-referenced) set between source and peer -- a +# repset the forwarding path and a future direct-peer path disagree on can +# permanently drop changes on cutover. DDL replication is disabled for the +# ALTER itself so the mismatch is real and local to n2. +# ============================================================================= +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := false)"; + +my $repset_mismatch_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_repset_mismatch'; +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $repset_mismatch_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn), + '--bidirectional rejects mismatched replication-set flags between source and peer'); +remove_tree($repset_mismatch_datadir) if -d $repset_mismatch_datadir; + +system_or_bail "$pg_bin/psql", '-q', '-p', $node_ports->[1], '-d', $dbname, '-c', + "SET spock.enable_ddl_replication = off; " . + "SELECT spock.repset_alter('default', replicate_truncate := true)"; + +# ============================================================================= +# TEST: a failed base backup leaves the source slot recoverable via +# --cleanup, even though the real manifest was never written -- a +# pending-cleanup sidecar is persisted right after source slot creation, +# before the backup even starts, since data_dir must stay empty until +# pg_basebackup runs. +# ============================================================================= +my $failed_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_failed'; +my $pending_sidecar = "${failed_datadir}.spock_bidir_pending.json"; +remove_tree($failed_datadir) if -d $failed_datadir; +unlink($pending_sidecar) if -f $pending_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $failed_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_xyz'), + 'a broken --extra-basebackup-args makes the base backup fail'); + +ok(-f $pending_sidecar, 'pending-cleanup sidecar written before the failed backup'); + +my $sidecar_mode = (stat($pending_sidecar))[2] & 07777; +is(sprintf('%04o', $sidecar_mode), '0600', + 'pending-cleanup sidecar is mode 0600 (may carry a DSN password)'); + +my $slot_after_failed_backup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +ok($slot_after_failed_backup >= 1, + 'source slot still exists on n1 after the failed backup (orphaned)'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $failed_datadir, + ], + '--cleanup --force recovers via the pending sidecar (no real manifest exists)' +); + +my $slot_after_sidecar_cleanup = scalar_query(1, + "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); +is($slot_after_sidecar_cleanup, '0', + 'source slot removed from n1 via sidecar-based cleanup'); + +ok(!-f $pending_sidecar, 'pending-cleanup sidecar removed after cleanup'); +remove_tree($failed_datadir) if -d $failed_datadir; + +# ============================================================================= +# TEST: an incomplete cleanup (source unreachable) exits non-zero and keeps +# the pending sidecar so it can be retried, instead of unconditionally +# deleting the only retry record. +# ============================================================================= +my $retry_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_retry'; +my $retry_sidecar = "${retry_datadir}.spock_bidir_pending.json"; +remove_tree($retry_datadir) if -d $retry_datadir; +unlink($retry_sidecar) if -f $retry_sidecar; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $retry_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--extra-basebackup-args', '--waldir=/nonexistent_pr3_test_waldir_retry'), + 'a broken backup orphans a slot for the retry-cleanup test'); +ok(-f $retry_sidecar, 'pending sidecar written for the retry-cleanup test'); + +my $n1_datadir = $config->{node_datadirs}->[0]; +system_or_bail "$pg_bin/pg_ctl", 'stop', '-D', $n1_datadir, '-m', 'fast'; + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir), + '--cleanup exits non-zero when the source is unreachable'); +ok(-f $retry_sidecar, + 'pending sidecar retained after an incomplete cleanup (retryable)'); + +system_or_bail "$pg_bin/pg_ctl", 'start', '-D', $n1_datadir, + '-l', "$config->{log_dir}/n1_retry_restart.log"; +ok(wait_for_pg_ready($host, $node_ports->[0], $pg_bin, 30), + 'n1 postgres is running again'); + +command_ok( + [ $SCS_BIN, + '--bidirectional', + '--cleanup', + '--force', + '--pgdata', $retry_datadir, + ], + '--cleanup --force succeeds once the source is reachable again' +); +ok(!-f $retry_sidecar, + 'pending sidecar removed once cleanup actually completed'); +remove_tree($retry_datadir) if -d $retry_datadir; + +# ============================================================================= +# CLEANUP +# ============================================================================= +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_tbl"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_part"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP TABLE IF EXISTS pr3_test_cols"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "DROP SEQUENCE IF EXISTS pr3_test_seq"; +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + q(DROP SEQUENCE IF EXISTS "weird's_seq"); +system_maybe "$pg_bin/psql", '-q', '-p', $node_ports->[0], '-d', $dbname, '-c', + "SELECT spock.repset_drop('pr3_test_repset')"; +unlink($n3_conf) if -f $n3_conf; +destroy_cluster('Cleanup'); diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index bff537361..96ed0ac18 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -52,8 +53,12 @@ #include "access/timeline.h" #include "access/xlog_internal.h" #include "catalog/pg_control.h" +#include "common/controldata_utils.h" +#include "common/file_utils.h" #include "common/jsonapi.h" +#include "common/logging.h" #include "mb/pg_wchar.h" +#include "port.h" #include "spock_fe.h" @@ -87,10 +92,57 @@ typedef struct BidirectionalState int max_wait; /* default 0 = unbounded */ char *source_slot_name; char *source_origin_name; + char *source_restore_lsn; /* recovery target LSN; consumed by the + * disabled-first catchup sub_create */ + char *node_dsn; /* DSN registered via spock.node_create(); + * the address peers use to connect back to + * this node. Derived from --subscriber-dsn. */ bool cleanup_mode; + bool force_cleanup; /* --force: also remove the data directory + * on --cleanup, not just remote state */ char *manifest_path; } BidirectionalState; +/* + * Replication-set / table-membership / sequence state captured from the + * source's catalog before DROP EXTENSION spock removes it. Utility-side + * memory only; never written to the manifest. + */ +typedef struct RepsetCapture +{ + char *set_name; + bool replicate_insert; + bool replicate_update; + bool replicate_delete; + bool replicate_truncate; +} RepsetCapture; + +typedef struct RepsetTableCapture +{ + char *set_name; + char *qualified_table; /* rts.set_reloid::regclass */ + char *columns; /* rts.set_att_list, NULL if all columns */ + char *row_filter; /* pg_get_expr(...), NULL if none */ +} RepsetTableCapture; + +typedef struct SequenceCapture +{ + char *set_name; + char *qualified_seq; + int64 last_value; + bool is_called; +} SequenceCapture; + +typedef struct CatalogCapture +{ + RepsetCapture *repsets; + int num_repsets; + RepsetTableCapture *tables; + int num_tables; + SequenceCapture *sequences; + int num_sequences; +} CatalogCapture; + typedef enum { VERBOSITY_NORMAL, VERBOSITY_VERBOSE, @@ -119,8 +171,10 @@ static int run_pg_ctl(const char *arg); static void validate_extra_basebackup_args(const char *args); static void run_basebackup(const char *provider_connstr, const char *data_dir, const char *extra_basebackup_args); +static char *reset_subscriber_sysid(const char *data_dir); +static void run_pg_resetwal(const char *data_dir); static void wait_postmaster_connection(const char *connstr); -static void wait_primary_connection(const char *connstr); +static void wait_primary_connection(const char *connstr, int stall_timeout, int max_wait); static void wait_postmaster_shutdown(void); static char *validate_replication_set_input(char *replication_sets); @@ -170,17 +224,36 @@ static char *generate_restore_point_name(void); static int discover_peer_nodes(PGconn *source_conn, const char *source_node_name, const char *subscriber_name, const char *dbname, PeerNodeInfo **peers_out); -static void check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers); +static void check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers); +static void check_spock_version_at_least_6(PGconn *conn, const char *node_label); +static void check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes); +static void check_peer_identity(PGconn *peer_conn, const char *expected_name); +static void check_replication_set_equivalence(PGconn *source_conn, + const char *source_node_name, + PeerNodeInfo *peers, int num_peers); static void write_manifest(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn); static bool read_manifest(const char *manifest_path, BidirectionalState *state, char **subscriber_name_out, char **dbname_out, char **source_dsn_out); -static void cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir); static void append_json_string(PQExpBuffer buf, const char *str); +static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname); +static void check_no_native_subscriptions(PGconn *conn); +static void capture_catalog_state(PGconn *conn, Oid source_nodeid, + CatalogCapture *capture); +static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); +static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); +static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void set_readonly_local(PGconn *conn); +static Oid get_local_node_id(PGconn *conn); + static PGconn * connectdb(const char *connstr) { @@ -195,7 +268,7 @@ connectdb(const char *connstr) void signal_handler(int sig) { - if (sig == SIGINT) + if (sig == SIGINT || sig == SIGTERM) { die(_("\nCanceling...\n")); } @@ -313,36 +386,524 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, } /* - * Verify that the source cluster and all peers meet the requirements for - * a bidirectional join: Spock >= 6.0.0, track_commit_timestamp on, no - * pending DDL, full-mesh topology, and peer connectivity. + * Verify Spock version on conn (the source or a peer): an old apply + * worker would advance the wrong-named origin, so this must be checked + * everywhere up front, not just on the source. */ static void -check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) +check_spock_version_at_least_6(PGconn *conn, const char *node_label) { PGresult *res; - int i; - /* Spock version gate: require >= 6.0.0 */ - res = PQexec(source_conn, - "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + res = PQexec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not query Spock extension version: %s"), - PQerrorMessage(source_conn)); + { + PQclear(res); + die(_("could not query Spock extension version on \"%s\": %s"), + node_label, PQerrorMessage(conn)); + } if (PQntuples(res) == 0) - die(_("Spock extension is not installed on the source node")); + { + PQclear(res); + die(_("Spock extension is not installed on \"%s\""), node_label); + } { const char *ver = PQgetvalue(res, 0, 0); int major = 0; + /* + * die() exits immediately -- ver points inside res, so it must + * not be PQclear()'d first (that would be a use-after-free when + * die()'s own formatting reads ver). + */ if (sscanf(ver, "%d.", &major) < 1) - die(_("could not parse Spock version \"%s\""), ver); + die(_("could not parse Spock version \"%s\" on \"%s\""), ver, node_label); if (major < 6) - die(_("Spock version %s on source is too old for bidirectional " - "join; require >= 6.0.0"), ver); + die(_("Spock version %s on \"%s\" is too old for bidirectional " + "join; require >= 6.0.0"), ver, node_label); + } + PQclear(res); +} + +/* + * Validate the actual directed subscription graph from one node's own + * catalog, not just a count: exactly one healthy (status = 'replicating') + * subscription from every other node in the set, no self-reference, no + * edge from outside the set, and no duplicate edge from the same origin + * regardless of status. + */ +static void +check_mesh_edges(PGconn *conn, const char *this_node_name, + char **all_names, int total_nodes) +{ + PGresult *res; + bool *healthy; + int *edge_count; + int i; + + /* + * spock.sub_show_status() (not raw sub_enabled) so "enabled" also + * means "actually replicating" -- a worker that's down or still + * initializing must not satisfy the mesh. + */ + res = PQexec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check subscription topology on \"%s\": %s"), + this_node_name, PQerrorMessage(conn)); + } + + healthy = pg_malloc0(total_nodes * sizeof(bool)); + edge_count = pg_malloc0(total_nodes * sizeof(int)); + + for (i = 0; i < PQntuples(res); i++) + { + const char *origin_name = PQgetvalue(res, i, 0); + const char *status = PQgetvalue(res, i, 1); + int idx = -1; + int k; + + if (strcmp(origin_name, this_node_name) == 0) + die(_("node \"%s\" has a subscription whose origin is itself; " + "corrupt or misconfigured topology"), this_node_name); + + for (k = 0; k < total_nodes; k++) + { + if (strcmp(all_names[k], origin_name) == 0) + { + idx = k; + break; + } + } + if (idx == -1) + die(_("node \"%s\" has a subscription from \"%s\", which is not " + "part of the discovered node set; partial-mesh or " + "unknown-node topologies are not supported"), + this_node_name, origin_name); + + /* + * Count regardless of status: an extra disabled duplicate from + * the same origin is still a duplicate edge. + */ + edge_count[idx]++; + if (edge_count[idx] > 1) + die(_("node \"%s\" has more than one subscription from \"%s\" " + "(status \"%s\"); duplicate edges are not supported"), + this_node_name, origin_name, status); + + if (strcmp(status, "replicating") == 0) + healthy[idx] = true; + } + PQclear(res); + + for (i = 0; i < total_nodes; i++) + { + if (strcmp(all_names[i], this_node_name) == 0) + continue; /* skip self */ + if (!healthy[i]) + { + pg_free(healthy); + pg_free(edge_count); + die(_("node \"%s\" has no healthy (status = 'replicating') " + "subscription from \"%s\"; full-mesh topology of live " + "replication is required for bidirectional join"), + this_node_name, all_names[i]); + } + } + pg_free(healthy); + pg_free(edge_count); +} + +/* + * Confirm the peer identifies itself as the name it was discovered + * under, so a node-name collision or wrong DSN can't silently validate + * the mesh against the wrong node. + */ +static void +check_peer_identity(PGconn *peer_conn, const char *expected_name) +{ + PGresult *res; + + res = PQexec(peer_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify identity of peer \"%s\": %s"), + expected_name, PQerrorMessage(peer_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), expected_name) != 0) + { + char *actual_name = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("peer discovered as \"%s\" identifies itself as \"%s\" once " + "connected; node-name/identity mismatch, refusing to trust " + "this topology\n"), expected_name, actual_name); + } + PQclear(res); +} + +/* + * Build a canonical, comparable fingerprint of one replication set as + * defined by its owning node: operation flags, each member table (sorted, + * with column list, row filter, and schema), then each member sequence + * (sorted). Scoped by node_id since spock.replication_set is keyed + * UNIQUE(set_nodeid, set_name) -- a set replicated via DDL becomes the + * replaying node's own row, not an echo. selected_filter restricts this + * to sets actually referenced by a subscription's sub_replication_sets, + * since unused/scratch repsets can legitimately differ between nodes. + */ +typedef struct RepsetFingerprintEntry +{ + char *set_name; + char *fingerprint; +} RepsetFingerprintEntry; + +static void +compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filter, + RepsetFingerprintEntry **out, int *nout) +{ + PGresult *res; + RepsetFingerprintEntry *entries; + int n; + int i; + PQExpBuffer query = createPQExpBuffer(); + + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_nodeid = %u" + " AND (%s)" + " ORDER BY set_name", node_id, selected_filter); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + destroyPQExpBuffer(query); + die(_("could not fingerprint replication sets: %s\n"), PQerrorMessage(conn)); + } + + n = PQntuples(res); + entries = pg_malloc0(n * sizeof(RepsetFingerprintEntry)); + + for (i = 0; i < n; i++) + { + PQExpBuffer fp = createPQExpBuffer(); + PGresult *tres; + PGresult *sres; + int j; + + entries[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + appendPQExpBuffer(fp, "flags=%s%s%s%s;", + PQgetvalue(res, i, 1)[0] == 't' ? "i" : "", + PQgetvalue(res, i, 2)[0] == 't' ? "u" : "", + PQgetvalue(res, i, 3)[0] == 't' ? "d" : "", + PQgetvalue(res, i, 4)[0] == 't' ? "t" : ""); + + printfPQExpBuffer(query, + "SELECT rts.set_reloid::regclass::text, rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rts.set_reloid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + tres = PQexec(conn, query->data); + if (PQresultStatus(tres) != PGRES_TUPLES_OK) + { + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint table memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + + for (j = 0; j < PQntuples(tres); j++) + { + const char *qualified_table = PQgetvalue(tres, j, 0); + PGresult *cres; + PQExpBuffer schema_query = createPQExpBuffer(); + int k; + + appendPQExpBuffer(fp, "tbl=%s|cols=%s|filter=%s|schema=(", + qualified_table, + PQgetisnull(tres, j, 1) ? "*" : PQgetvalue(tres, j, 1), + PQgetisnull(tres, j, 2) ? "-" : PQgetvalue(tres, j, 2)); + + /* + * Schema fingerprint: relation kind and replica identity, + * then per-column name, type, typmod (varchar(10) vs + * varchar(100) is otherwise invisible), collation, + * nullability, and generated/identity status -- so a + * divergent column or relation definition is caught even if + * repset membership itself matches. + */ + printfPQExpBuffer(schema_query, + "SELECT relkind::text, relreplident::text" + " FROM pg_class WHERE oid = %s::regclass", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = PQexec(conn, schema_query->data); + if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(schema_query); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint relation kind of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + appendPQExpBuffer(fp, "relkind=%s|replident=%s|", + PQgetvalue(cres, 0, 0), PQgetvalue(cres, 0, 1)); + PQclear(cres); + + printfPQExpBuffer(schema_query, + "SELECT a.attname, a.atttypid::regtype::text, a.atttypmod," + " a.attnotnull, a.attidentity, a.attgenerated," + " COALESCE(co.collname, '')" + " FROM pg_attribute a" + " LEFT JOIN pg_collation co ON co.oid = a.attcollation" + " WHERE a.attrelid = %s::regclass AND a.attnum > 0" + " AND NOT a.attisdropped ORDER BY a.attnum", + PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); + cres = PQexec(conn, schema_query->data); + destroyPQExpBuffer(schema_query); + if (PQresultStatus(cres) != PGRES_TUPLES_OK) + { + PQclear(cres); + PQclear(tres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint schema of \"%s\": %s\n"), + qualified_table, PQerrorMessage(conn)); + } + for (k = 0; k < PQntuples(cres); k++) + appendPQExpBuffer(fp, "%s%s:%s:%s:notnull=%s:ident=%s:gen=%s:coll=%s", + k > 0 ? "," : "", + PQgetvalue(cres, k, 0), + PQgetvalue(cres, k, 1), + PQgetvalue(cres, k, 2), + PQgetvalue(cres, k, 3), + PQgetvalue(cres, k, 4), + PQgetvalue(cres, k, 5), + PQgetvalue(cres, k, 6)); + appendPQExpBufferStr(fp, ");"); + PQclear(cres); + } + PQclear(tres); + + printfPQExpBuffer(query, + "SELECT rss.set_seqoid::regclass::text" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u AND rs.set_name = %s" + " ORDER BY rss.set_seqoid::regclass::text", + node_id, + PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); + sres = PQexec(conn, query->data); + if (PQresultStatus(sres) != PGRES_TUPLES_OK) + { + PQclear(sres); + PQclear(res); + destroyPQExpBuffer(query); + destroyPQExpBuffer(fp); + die(_("could not fingerprint sequence memberships for set \"%s\": %s\n"), + entries[i].set_name, PQerrorMessage(conn)); + } + for (j = 0; j < PQntuples(sres); j++) + appendPQExpBuffer(fp, "seq=%s;", PQgetvalue(sres, j, 0)); + PQclear(sres); + + entries[i].fingerprint = pg_strdup(fp->data); + destroyPQExpBuffer(fp); + } + PQclear(res); + destroyPQExpBuffer(query); + + *out = entries; + *nout = n; +} + +static void +free_repset_fingerprints(RepsetFingerprintEntry *entries, int n) +{ + int i; + + for (i = 0; i < n; i++) + { + pg_free(entries[i].set_name); + pg_free(entries[i].fingerprint); + } + pg_free(entries); +} + +/* + * Build a SQL boolean expression ("set_name IN (...)") over the union of + * every replication set actually referenced by conn's own subscriptions + * (sub_replication_sets), rather than every set that happens to exist + * locally. Caller frees the result. + */ +static char * +build_selected_set_name_filter(PGconn *conn) +{ + PGresult *res; + PQExpBuffer filter; + char *result; + int i; + + res = PQexec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine selected replication sets: %s\n"), PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "verify replication-set equivalence\n")); + } + + filter = createPQExpBuffer(); + appendPQExpBufferStr(filter, "set_name IN ("); + for (i = 0; i < PQntuples(res); i++) + { + char *name = PQgetvalue(res, i, 0); + + appendPQExpBuffer(filter, "%s%s", i > 0 ? ", " : "", + PQescapeLiteral(conn, name, strlen(name))); } + appendPQExpBufferStr(filter, ")"); PQclear(res); + result = pg_strdup(filter->data); + destroyPQExpBuffer(filter); + return result; +} + +/* + * The forwarding path (peer -> source -> n3) and the future direct path + * (peer -> n3) must select exactly the same changes, or a change omitted + * on one path is lost once the direct subscription takes over. Compare + * every selected replication set's fingerprint between the source and + * each peer; reject any mismatch or missing/extra set on either side. + */ +static void +check_replication_set_equivalence(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + Oid source_nodeid = get_local_node_id(source_conn); + char *selected_filter = build_selected_set_name_filter(source_conn); + RepsetFingerprintEntry *source_fps; + int num_source_fps; + int i; + + compute_repset_fingerprints(source_conn, source_nodeid, selected_filter, + &source_fps, &num_source_fps); + + for (i = 0; i < num_peers; i++) + { + PGconn *peer_conn; + Oid peer_nodeid; + RepsetFingerprintEntry *peer_fps; + int num_peer_fps; + int j; + + peer_conn = PQconnectdb(peers[i].dsn); + if (PQstatus(peer_conn) != CONNECTION_OK) + die(_("cannot connect to peer \"%s\": %s"), + peers[i].node_name, PQerrorMessage(peer_conn)); + + peer_nodeid = get_local_node_id(peer_conn); + compute_repset_fingerprints(peer_conn, peer_nodeid, selected_filter, + &peer_fps, &num_peer_fps); + + /* + * die() exits immediately, so none of the branches below free + * source_fps/peer_fps before calling it -- freeing first and then + * still reading source_fps[j]/peer_fps[j] in the same die() call's + * arguments would be a use-after-free (the process is about to + * exit anyway; nothing else in this file frees before die() either). + */ + for (j = 0; j < num_source_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_peer_fps; k++) + { + if (strcmp(source_fps[j].set_name, peer_fps[k].set_name) != 0) + continue; + found = true; + if (strcmp(source_fps[j].fingerprint, peer_fps[k].fingerprint) != 0) + die(_("replication set \"%s\" differs between the source " + "and peer \"%s\" (membership, flags, columns, row " + "filter, or schema) -- the forwarding path and a " + "future direct peer subscription would not select " + "the same changes, risking permanently lost data " + "on cutover. Reconcile the definitions before " + "retrying.\n"), + source_fps[j].set_name, peers[i].node_name); + break; + } + if (!found) + die(_("replication set \"%s\" exists on the source but not " + "on peer \"%s\"\n"), source_fps[j].set_name, peers[i].node_name); + } + for (j = 0; j < num_peer_fps; j++) + { + int k; + bool found = false; + + for (k = 0; k < num_source_fps; k++) + if (strcmp(peer_fps[j].set_name, source_fps[k].set_name) == 0) + { + found = true; + break; + } + if (!found) + die(_("replication set \"%s\" exists on peer \"%s\" but not " + "on the source\n"), peer_fps[j].set_name, peers[i].node_name); + } + + free_repset_fingerprints(peer_fps, num_peer_fps); + PQfinish(peer_conn); + } + + free_repset_fingerprints(source_fps, num_source_fps); + pg_free(selected_filter); + (void) source_node_name; +} + +/* + * Verify that the source cluster and all peers meet the requirements for + * a bidirectional join: Spock >= 6.0.0 on every node, track_commit_timestamp + * on, no pending DDL, an actual full-mesh subscription graph (not just a + * count), replication-set/schema equivalence across the source and every + * peer, and peer connectivity. + */ +static void +check_preconditions(PGconn *source_conn, const char *source_node_name, + PeerNodeInfo *peers, int num_peers) +{ + PGresult *res; + int i; + int total_nodes = num_peers + 1; + char **all_names = pg_malloc(total_nodes * sizeof(char *)); + + all_names[0] = pg_strdup(source_node_name); + for (i = 0; i < num_peers; i++) + all_names[i + 1] = pg_strdup(peers[i].node_name); + + check_spock_version_at_least_6(source_conn, "source"); + /* track_commit_timestamp must be on at the source */ res = PQexec(source_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -352,36 +913,34 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) die(_("track_commit_timestamp must be on for bidirectional join (source)")); PQclear(res); - /* No pending DDL in spock.queue */ - res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.queue"); + /* + * All outbound replication caught up to the source's current WAL + * position -- i.e. nothing (DDL or data) still in flight to an + * existing peer. spock.queue's row count is not a usable signal here: + * queue_message() (spock_queue.c) only ever inserts into it, so its + * count is monotonically non-decreasing and is never zero on any node + * that has replicated so much as a single DDL statement. + */ + res = PQexec(source_conn, + "SELECT COUNT(*) FROM pg_replication_slots" + " WHERE slot_type = 'logical' AND plugin = 'spock_output'" + " AND (confirmed_flush_lsn IS NULL" + " OR confirmed_flush_lsn < pg_current_wal_lsn())"); if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not check spock.queue: %s"), + die(_("could not check replication slot lag: %s"), PQerrorMessage(source_conn)); if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) - die(_("pending DDL in spock.queue; wait for replication to drain " - "before joining")); + die(_("source has unreplicated changes pending to an existing peer; " + "wait for replication to drain before joining")); PQclear(res); - /* Full-mesh assertion: subscriptions on source == num_peers */ - res = PQexec(source_conn, "SELECT COUNT(*) FROM spock.subscription"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not count subscriptions: %s"), - PQerrorMessage(source_conn)); - { - int sub_count = atoi(PQgetvalue(res, 0, 0)); - - if (sub_count != num_peers) - die(_("source node has %d active subscription(s) but %d peer(s) " - "discovered; partial-mesh topologies are not supported"), - sub_count, num_peers); - } - PQclear(res); + /* Full-mesh directed-graph check, from the source's own perspective. */ + check_mesh_edges(source_conn, source_node_name, all_names, total_nodes); /* - * Per-peer: connectivity and track_commit_timestamp. - * - * Spock version is not checked on peers here; peer version checking is - * deferred to the subscription-setup phase. + * Per-peer: connectivity, Spock version, track_commit_timestamp, and + * the full-mesh directed-graph check from each peer's own perspective + * (a mesh that's only complete as seen from the source is not a mesh). */ for (i = 0; i < num_peers; i++) { @@ -395,11 +954,18 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) die(_("cannot connect to peer \"%s\": %s"), peers[i].node_name, PQerrorMessage(peer_conn)); + check_peer_identity(peer_conn, peers[i].node_name); + check_spock_version_at_least_6(peer_conn, peers[i].node_name); + res = PQexec(peer_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { + /* + * die() exits immediately -- PQerrorMessage() needs peer_conn + * still open, so PQfinish() must not run first (that would be + * a use-after-free when die()'s own formatting reads it). + */ PQclear(res); - PQfinish(peer_conn); die(_("could not check track_commit_timestamp on peer \"%s\": %s"), peers[i].node_name, PQerrorMessage(peer_conn)); } @@ -411,57 +977,219 @@ check_preconditions(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) "(peer \"%s\")"), peers[i].node_name); } PQclear(res); + + check_mesh_edges(peer_conn, peers[i].node_name, all_names, total_nodes); + PQfinish(peer_conn); } + /* Replication-set & schema equivalence: run once the mesh is sound. */ + check_replication_set_equivalence(source_conn, source_node_name, peers, num_peers); + + for (i = 0; i < total_nodes; i++) + pg_free(all_names[i]); + pg_free(all_names); + print_msg(VERBOSITY_NORMAL, _("Preconditions verified.\n")); } /* - * Write the bidirectional state manifest to state->manifest_path - * atomically (write to .tmp, then rename). The manifest is a simple - * hand-formatted JSON file, with string values escaped by - * append_json_string(). + * The physical-backup path runs once per data directory, so it requires + * exactly one spock-configured database on the source instance. Checked + * against actual spock configuration, not --databases/--provider-dsn, + * since the instance can host other unrelated databases. Fails closed: + * any database we cannot inspect aborts the run rather than being + * treated as spock-free. datallowconn is not used to skip databases -- + * a database with connections disabled can still hold spock catalog + * state -- only true templates are excluded. */ static void -write_manifest(BidirectionalState *state, const char *subscriber_name, - const char *dbname, const char *source_dsn) +check_single_spock_database(PGconn *conn, const char *base_prov_connstr, + const char *current_dbname) { - PQExpBuffer buf = createPQExpBuffer(); - char tmp_path[MAXPGPATH]; - FILE *f; + PGresult *res; int i; + PQExpBuffer others = createPQExpBuffer(); + int other_count = 0; - snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); + res = PQexec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not list databases on source: %s\n"), PQerrorMessage(conn)); + } - appendPQExpBufferStr(buf, "{\n"); - appendPQExpBufferStr(buf, " \"version\": 1,\n"); + for (i = 0; i < PQntuples(res); i++) + { + char *dbname = PQgetvalue(res, i, 0); + char *db_connstr; + PGconn *db_conn; + PGresult *ext_res; + PGresult *node_res; - appendPQExpBufferStr(buf, " \"subscriber_name\": \""); - append_json_string(buf, subscriber_name); - appendPQExpBufferStr(buf, "\",\n"); + if (strcmp(dbname, current_dbname) == 0) + continue; - appendPQExpBufferStr(buf, " \"dbname\": \""); - append_json_string(buf, dbname); - appendPQExpBufferStr(buf, "\",\n"); + db_connstr = get_connstr((char *) base_prov_connstr, dbname); + db_conn = PQconnectdb(db_connstr); + if (PQstatus(db_conn) != CONNECTION_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); - appendPQExpBufferStr(buf, " \"source_dsn\": \""); - append_json_string(buf, source_dsn); - appendPQExpBufferStr(buf, "\",\n"); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not connect to " + "\"%s\" to check: %s\n"), dbname, errmsg); + } - appendPQExpBufferStr(buf, " \"source_slot_name\": \""); - if (state->source_slot_name) - append_json_string(buf, state->source_slot_name); - appendPQExpBufferStr(buf, "\",\n"); + ext_res = PQexec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); - appendPQExpBufferStr(buf, " \"source_origin_name\": \""); - if (state->source_origin_name) - append_json_string(buf, state->source_origin_name); - appendPQExpBufferStr(buf, "\",\n"); + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on the " + "source has spock configured, but could not query \"%s\": " + "%s\n"), dbname, errmsg); + } - appendPQExpBufferStr(buf, " \"peers\": [\n"); - for (i = 0; i < state->num_peers; i++) - { + if (PQntuples(ext_res) > 0) + { + node_res = PQexec(db_conn, "SELECT 1 FROM spock.local_node"); + if (PQresultStatus(node_res) != PGRES_TUPLES_OK) + { + char *errmsg = pg_strdup(PQerrorMessage(db_conn)); + + PQclear(node_res); + PQclear(ext_res); + PQfinish(db_conn); + PQclear(res); + die(_("--bidirectional requires proving no other database on " + "the source has spock configured, but could not query " + "spock.local_node in \"%s\": %s\n"), dbname, errmsg); + } + + if (PQntuples(node_res) > 0) + { + appendPQExpBuffer(others, "%s%s", other_count ? ", " : "", dbname); + other_count++; + } + PQclear(node_res); + } + PQclear(ext_res); + PQfinish(db_conn); + } + PQclear(res); + + if (other_count > 0) + die(_("--bidirectional requires exactly one spock-configured database " + "on the source instance; also found spock configured on: %s\n"), + others->data); + + destroyPQExpBuffer(others); +} + +/* + * A physical base backup copies native (non-spock) logical subscriptions + * too, which DROP EXTENSION spock doesn't touch. Once n3 is promoted and + * restarted, an enabled native subscription would start consuming from + * its provider as a second, unintended consumer. pg_subscription is a + * shared catalog, so one query sees every database's rows. + */ +static void +check_no_native_subscriptions(PGconn *conn) +{ + PGresult *res; + + res = PQexec(conn, + "SELECT s.subname, d.datname" + " FROM pg_subscription s" + " JOIN pg_database d ON d.oid = s.subdbid" + " WHERE s.subenabled"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check for native logical subscriptions on the " + "source instance: %s\n"), PQerrorMessage(conn)); + } + + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; + + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s (database %s)", + PQgetvalue(res, i, 0), PQgetvalue(res, i, 1)); + + PQclear(res); + die(_("--bidirectional requires no enabled native (non-spock) logical " + "subscriptions anywhere on the source instance -- a physical " + "backup would copy them, and they would start consuming on " + "the new node as an unintended second consumer once " + "promoted: %s\nDisable or drop these subscriptions before " + "retrying.\n"), list->data); + } + PQclear(res); +} + +/* + * Write the bidirectional state manifest to state->manifest_path + * atomically (write to .tmp, then rename). The manifest is a simple + * hand-formatted JSON file, with string values escaped by + * append_json_string(). + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + char tmp_path[MAXPGPATH]; + int i; + + snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); + + appendPQExpBufferStr(buf, "{\n"); + appendPQExpBufferStr(buf, " \"version\": 1,\n"); + + appendPQExpBufferStr(buf, " \"subscriber_name\": \""); + append_json_string(buf, subscriber_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"dbname\": \""); + append_json_string(buf, dbname); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_dsn\": \""); + append_json_string(buf, source_dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_slot_name\": \""); + if (state->source_slot_name) + append_json_string(buf, state->source_slot_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_origin_name\": \""); + if (state->source_origin_name) + append_json_string(buf, state->source_origin_name); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"source_restore_lsn\": \""); + if (state->source_restore_lsn) + append_json_string(buf, state->source_restore_lsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"node_dsn\": \""); + if (state->node_dsn) + append_json_string(buf, state->node_dsn); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBufferStr(buf, " \"peers\": [\n"); + for (i = 0; i < state->num_peers; i++) + { PeerNodeInfo *p = &state->peers[i]; bool last = (i == state->num_peers - 1); @@ -481,34 +1209,83 @@ write_manifest(BidirectionalState *state, const char *subscriber_name, appendPQExpBufferStr(buf, " \"peer_slot_name\": \""); append_json_string(buf, p->slot_name); - appendPQExpBufferStr(buf, "\"\n"); + appendPQExpBufferStr(buf, "\",\n"); + + appendPQExpBuffer(buf, " \"disabled_sub_created\": %s,\n", + p->disabled_sub_created ? "true" : "false"); + appendPQExpBuffer(buf, " \"slot_created\": %s,\n", + p->slot_created ? "true" : "false"); + appendPQExpBuffer(buf, " \"reverse_sub_created\": %s\n", + p->reverse_sub_created ? "true" : "false"); appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); } appendPQExpBufferStr(buf, " ]\n"); appendPQExpBufferStr(buf, "}\n"); - f = fopen(tmp_path, "w"); - if (f == NULL) - die(_("could not create manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - - if (fwrite(buf->data, 1, buf->len, f) != buf->len) - { - fclose(f); - unlink(tmp_path); - die(_("could not write manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - } - if (fclose(f) != 0) + /* + * The manifest can embed a password (source_dsn, node_dsn), so create + * with mode 0600 up front, not a post-hoc chmod. O_EXCL|O_NOFOLLOW + * refuses to write through a pre-existing file or planted symlink, + * except a leftover .tmp from a previous crashed run. + */ { - unlink(tmp_path); - die(_("could not close manifest file \"%s\": %s"), - tmp_path, strerror(errno)); + int fd; + ssize_t written; + + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + if (fd < 0 && errno == EEXIST) + { + if (unlink(tmp_path) != 0) + die(_("could not remove stale manifest temp file \"%s\": %s"), + tmp_path, strerror(errno)); + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + } + if (fd < 0) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + + written = write(fd, buf->data, buf->len); + if (written < 0 || (size_t) written != buf->len) + { + close(fd); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + + /* + * fsync, rename, then fsync the directory -- a crash right after + * this returns must not lose the only cleanup record for the + * source slot created just before it. + */ + if (fsync(fd) != 0) + { + close(fd); + unlink(tmp_path); + die(_("could not fsync manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (close(fd) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, state->manifest_path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + state->manifest_path, strerror(errno)); + + /* + * fsync_parent_path() already treats "filesystem doesn't support + * directory fsync" as success internally, so a nonzero return + * here is a genuine failure that can orphan the source slot + * after a crash. Fatal, like the durability steps above. + */ + if (fsync_parent_path(state->manifest_path) != 0) + die(_("could not fsync directory containing \"%s\": %s\n"), + state->manifest_path, strerror(errno)); } - if (rename(tmp_path, state->manifest_path) != 0) - die(_("could not rename manifest to \"%s\": %s"), - state->manifest_path, strerror(errno)); destroyPQExpBuffer(buf); } @@ -536,6 +1313,9 @@ typedef struct ManifestParseState char *peer_dsn; char *peer_sub_name; char *peer_slot_name; + bool peer_disabled_sub_created; + bool peer_slot_created; + bool peer_reverse_sub_created; int peer_capacity; } ManifestParseState; @@ -569,8 +1349,12 @@ manifest_object_end(void *st) s->bidir->peers[i].dsn = s->peer_dsn; s->bidir->peers[i].sub_name = s->peer_sub_name; s->bidir->peers[i].slot_name = s->peer_slot_name; + s->bidir->peers[i].disabled_sub_created = s->peer_disabled_sub_created; + s->bidir->peers[i].slot_created = s->peer_slot_created; + s->bidir->peers[i].reverse_sub_created = s->peer_reverse_sub_created; s->bidir->num_peers++; s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; + s->peer_disabled_sub_created = s->peer_slot_created = s->peer_reverse_sub_created = false; s->in_peer_obj = false; } s->depth--; @@ -617,7 +1401,32 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) { ManifestParseState *s = (ManifestParseState *) st; - if (s->cur_field == NULL || tokentype != JSON_TOKEN_STRING) + if (s->cur_field == NULL) + { + pg_free(token); + return JSON_SUCCESS; + } + + /* + * Per-peer creation-state flags are JSON booleans, not strings -- + * handle them before the string-only fields below (which free and + * ignore anything that isn't JSON_TOKEN_STRING). + */ + if (s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, "disabled_sub_created") == 0) + s->peer_disabled_sub_created = value; + else if (strcmp(s->cur_field, "slot_created") == 0) + s->peer_slot_created = value; + else if (strcmp(s->cur_field, "reverse_sub_created") == 0) + s->peer_reverse_sub_created = value; + pg_free(token); + return JSON_SUCCESS; + } + + if (tokentype != JSON_TOKEN_STRING) { pg_free(token); return JSON_SUCCESS; @@ -636,6 +1445,10 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) s->bidir->source_slot_name = token; else if (strcmp(s->cur_field, "source_origin_name") == 0) s->bidir->source_origin_name = token; + else if (strcmp(s->cur_field, "source_restore_lsn") == 0) + s->bidir->source_restore_lsn = token; + else if (strcmp(s->cur_field, "node_dsn") == 0) + s->bidir->node_dsn = token; else pg_free(token); } @@ -735,12 +1548,16 @@ read_manifest(const char *manifest_path, BidirectionalState *state, /* * Idempotently remove bidirectional join state from all reachable nodes. - * Connects to the source and each peer, drops replication slots and - * reverse subscriptions created during a previous join attempt. All - * operations are best-effort: connectivity failures are logged as - * warnings rather than being fatal. + * Connects to the source and each peer; drops replication slots and + * reverse subscriptions created during a previous join attempt. + * Connectivity and drop failures are logged as warnings, not fatal, so + * cleanup attempts every remaining resource -- but each failure is + * tracked, and the function returns true only if every recorded resource + * was confirmed gone. The manifest/sidecar record (the only way to + * retry) is removed only on a true return; an incomplete cleanup keeps + * it and the caller exits non-zero. */ -static void +static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir) @@ -749,6 +1566,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, PGresult *res; PQExpBuffer query = createPQExpBuffer(); int i; + bool fully_cleaned = true; print_msg(VERBOSITY_NORMAL, _("Cleaning up partial bidirectional join state ...\n")); @@ -756,10 +1574,14 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) { - print_msg(VERBOSITY_NORMAL, - _("warning: cannot connect to source node; skipping " - "source-side cleanup: %s\n"), - PQerrorMessage(source_conn)); + if (state->source_slot_name && state->source_slot_name[0]) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to source node; slot %s " + "may still exist: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } PQfinish(source_conn); source_conn = NULL; } @@ -773,10 +1595,20 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " WHERE slot_name = '%s'", state->source_slot_name); res = PQexec(source_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + if (PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped source slot %s\n"), + state->source_slot_name); + } + else + { print_msg(VERBOSITY_NORMAL, - _(" dropped source slot %s\n"), - state->source_slot_name); + _("warning: could not drop source slot %s: %s\n"), + state->source_slot_name, PQerrorMessage(source_conn)); + fully_cleaned = false; + } PQclear(res); } @@ -790,18 +1622,29 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, if (!peer->dsn || !peer->dsn[0]) continue; + /* + * Only attempt to drop -- and only require connectivity for -- + * resources this run actually recorded as created. Slot/sub names + * are deterministic, not per-run unique, so --cleanup must not + * touch a same-named resource from an unrelated join, nor report + * "incomplete" over a peer that was never touched. + */ + if (!peer->slot_created && !peer->reverse_sub_created) + continue; + peer_conn = PQconnectdb(peer->dsn); if (PQstatus(peer_conn) != CONNECTION_OK) { print_msg(VERBOSITY_NORMAL, - _("warning: cannot connect to peer \"%s\"; skipping " - "peer-side cleanup: %s\n"), + _("warning: cannot connect to peer \"%s\"; its slot/" + "subscription may still exist: %s\n"), peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; PQfinish(peer_conn); continue; } - if (peer->slot_name && peer->slot_name[0]) + if (peer->slot_created && peer->slot_name && peer->slot_name[0]) { printfPQExpBuffer(query, "SELECT pg_drop_replication_slot(slot_name)" @@ -809,25 +1652,49 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " WHERE slot_name = '%s'", peer->slot_name); res = PQexec(peer_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) > 0) + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + if (PQntuples(res) > 0) + print_msg(VERBOSITY_NORMAL, + _(" dropped peer slot %s on %s\n"), + peer->slot_name, peer->node_name); + } + else + { print_msg(VERBOSITY_NORMAL, - _(" dropped peer slot %s on %s\n"), - peer->slot_name, peer->node_name); + _("warning: could not drop peer slot %s on %s: %s\n"), + peer->slot_name, peer->node_name, + PQerrorMessage(peer_conn)); + fully_cleaned = false; + } PQclear(res); } /* - * Drop the reverse subscription (peer -> new subscriber) if it was - * created during a previous attempt. The sub_drop second argument - * is ifexists=true. + * Drop the reverse subscription (peer -> new subscriber) only if + * this run recorded having created it. The sub_drop second + * argument is ifexists=true, so an absent subscription is not an + * error -- only an actual query failure counts against + * fully_cleaned. */ - snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", - peer->node_name, subscriber_name); - printfPQExpBuffer(query, - "SELECT spock.sub_drop('%s', true)", - reverse_sub); - res = PQexec(peer_conn, query->data); - PQclear(res); + if (peer->reverse_sub_created) + { + snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", + peer->node_name, subscriber_name); + printfPQExpBuffer(query, + "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(peer_conn, reverse_sub, strlen(reverse_sub))); + res = PQexec(peer_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on " + "%s: %s\n"), + reverse_sub, peer->node_name, PQerrorMessage(peer_conn)); + fully_cleaned = false; + } + PQclear(res); + } PQfinish(peer_conn); print_msg(VERBOSITY_NORMAL, @@ -839,14 +1706,112 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, destroyPQExpBuffer(query); + /* + * The data directory a partial run may have created via basebackup. + * Never touch it without --force. + */ + if (data_dir != NULL && data_dir[0] && file_exists(data_dir)) + { + if (force_rm_datadir) + { + struct stat st; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s before removing it ...\n"), + data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + fully_cleaned = false; + } + } + else + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + } + } + + if (!fully_cleaned) + { + print_msg(VERBOSITY_NORMAL, + _("Cleanup incomplete: some resource(s) above could not be " + "confirmed removed. Keeping the manifest/sidecar record " + "so --cleanup can be retried.\n")); + return false; + } + + /* + * Every remote/local resource above was confirmed gone; now remove the + * retry record(s) themselves. An unexpected removal failure here + * (anything but ENOENT, i.e. already gone) must also flip + * fully_cleaned -- otherwise the caller reports success and exits 0 + * while a stale record that still references now-removed resources + * lingers on disk, which a later --cleanup could misread as current. + */ if (state->manifest_path && state->manifest_path[0]) { - unlink(state->manifest_path); + if (unlink(state->manifest_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed manifest %s\n"), state->manifest_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove manifest %s: %s\n"), + state->manifest_path, strerror(errno)); + fully_cleaned = false; + } + } + + /* + * Also remove any pending-cleanup sidecar, even if it wasn't the file + * that drove this cleanup: a stale one left behind by an earlier run + * whose own sidecar-unlink failed could otherwise be misread as + * current by a later --cleanup once the manifest above is gone, + * reporting resources as still-pending that were, in fact, already + * confirmed removed here. + */ + if (data_dir != NULL && data_dir[0]) + { + char sidecar_path[MAXPGPATH]; + + snprintf(sidecar_path, MAXPGPATH, "%s.spock_bidir_pending.json", data_dir); + if (unlink(sidecar_path) == 0) + print_msg(VERBOSITY_NORMAL, + _(" removed pending sidecar %s\n"), sidecar_path); + else if (errno != ENOENT) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove pending sidecar %s: %s\n"), + sidecar_path, strerror(errno)); + fully_cleaned = false; + } + } + + if (!fully_cleaned) + { print_msg(VERBOSITY_NORMAL, - _(" removed manifest %s\n"), state->manifest_path); + _("Cleanup incomplete: the manifest or sidecar record could " + "not be removed even though every resource it tracked " + "was confirmed gone. Retry --cleanup to remove the " + "stale record.\n")); + return false; } print_msg(VERBOSITY_NORMAL, _("Cleanup complete.\n")); + return true; } @@ -884,6 +1849,8 @@ main(int argc, char **argv) char *extra_basebackup_args = NULL; BidirectionalState bidir = {0}; char bidir_manifest_path[MAXPGPATH] = {0}; + char bidir_pending_path[MAXPGPATH] = {0}; + CatalogCapture capture = {0}; static struct option long_options[] = { {"subscriber-name", required_argument, NULL, 'n'}, @@ -904,13 +1871,16 @@ main(int argc, char **argv) {"stall-timeout", required_argument, NULL, 13}, {"max-wait", required_argument, NULL, 14}, {"cleanup", no_argument, NULL, 15}, + {"force", no_argument, NULL, 16}, {NULL, 0, NULL, 0} }; argv0 = argv[0]; progname = get_progname(argv[0]); + pg_logging_init(argv[0]); start_time = time(NULL); signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); /* check for --help */ if (argc > 1) @@ -1009,6 +1979,9 @@ main(int argc, char **argv) case 15: bidir.cleanup_mode = true; break; + case 16: + bidir.force_cleanup = true; + break; default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -1036,11 +2009,18 @@ main(int argc, char **argv) if (bidir.cleanup_mode && !bidir.enabled) die(_("--cleanup requires --bidirectional.\n")); + if (bidir.force_cleanup && !bidir.cleanup_mode) + die(_("--force requires --cleanup.\n")); + if (!bidir.cleanup_mode && (!base_prov_connstr || !strlen(base_prov_connstr))) die(_("Provider connection string must be specified.\n")); - if (!bidir.enabled && !bidir.cleanup_mode && + if (!bidir.cleanup_mode && (!base_sub_connstr || !strlen(base_sub_connstr))) - die(_("Subscriber connection string must be specified.\n")); + die(_("Subscriber connection string must be specified: --subscriber-dsn " + "is used both for the tool's own connection to the newly " + "created node and, with --bidirectional, as the externally-" + "reachable address registered via spock.node_create() for " + "peers to connect back to it.\n")); if (apply_delay < 0) die(_("Apply delay cannot be negative.\n")); @@ -1057,6 +2037,14 @@ main(int argc, char **argv) snprintf(bidir_manifest_path, MAXPGPATH, "%s/spock_bidirectional_manifest.json", data_dir); bidir.manifest_path = bidir_manifest_path; + /* + * Sidecar path for the source slot orphan-protection record (see + * the write near source-slot creation below) -- lives next to, not + * inside, data_dir, since data_dir must still be empty when this is + * first written (pg_basebackup requires an empty target directory). + */ + snprintf(bidir_pending_path, MAXPGPATH, + "%s.spock_bidir_pending.json", data_dir); if (bidir.stall_timeout == 0) bidir.stall_timeout = 600; } @@ -1068,13 +2056,23 @@ main(int argc, char **argv) char *db = NULL; char *src_dsn = NULL; - if (!read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) - { - fprintf(stderr, _("No manifest found at %s; nothing to clean up.\n"), - bidir.manifest_path); - exit(0); - } - cleanup_partial_state(&bidir, sub_name, db, src_dsn, false); + if (read_manifest(bidir.manifest_path, &bidir, &sub_name, &db, &src_dsn)) + exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, + bidir.force_cleanup) ? 0 : 1); + + /* + * No full manifest -- basebackup may never have completed. Fall + * back to the pending-cleanup sidecar written right after source + * slot creation, so a slot orphaned by a failed/interrupted backup + * is still reachable by --cleanup. + */ + if (read_manifest(bidir_pending_path, &bidir, &sub_name, &db, &src_dsn)) + /* cleanup_partial_state() removes the sidecar itself on success. */ + exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, + bidir.force_cleanup) ? 0 : 1); + + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), + bidir.manifest_path, bidir_pending_path); exit(0); } @@ -1099,6 +2097,18 @@ main(int argc, char **argv) database_list[0] = dbname; } + /* + * Single database only: all join state is per-database, and the + * physical-backup/recovery path operates on one data directory. + * Reject a multi-database request rather than silently joining only + * database_list[0]. Separate from check_single_spock_database() + * below, which checks the instance for spock on other databases. + */ + if (bidir.enabled && n_databases > 1) + die(_("--bidirectional supports a single database only; " + "%d were named via --databases/--provider-dsn.\n"), + n_databases); + slot_names = palloc(n_databases * sizeof(char *)); /* @@ -1153,26 +2163,66 @@ main(int argc, char **argv) remote_info = get_remote_info(provider_conn); /* - * --bidirectional: discover peers, verify preconditions, write the - * manifest, then exit. The rest of the join resumes from this - * manifest once the physical backup has been taken and the - * subscriber is running. + * --bidirectional: discover peers, verify preconditions, then + * continue into the physical-backup pipeline below using the + * "sub__" slot naming convention. Manifest + * write is deferred until after the basebackup; see the comment + * there. */ if (bidir.enabled) { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + char *source_sub_name; + bidir.num_peers = discover_peer_nodes(provider_conn, remote_info->node_name, subscriber_name, db, &bidir.peers); - check_preconditions(provider_conn, bidir.peers, bidir.num_peers); - write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + check_preconditions(provider_conn, remote_info->node_name, + bidir.peers, bidir.num_peers); + check_single_spock_database(provider_conn, base_prov_connstr, db); + check_no_native_subscriptions(provider_conn); + use_existing_data_dir = check_data_dir(data_dir, remote_info); + if (use_existing_data_dir) + { + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + } + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, remote_info->node_name); + source_sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + print_msg(VERBOSITY_NORMAL, - _("Bidirectional plumbing complete: %d peer(s) discovered, " - "preconditions OK, manifest written to %s.\n"), - bidir.num_peers, bidir.manifest_path); + _("Creating source replication slot in database %s ...\n"), db); + bidir.source_slot_name = initialize_replication_slot(provider_conn, + remote_info->dbname, + remote_info->node_name, + source_sub_name, + drop_slot_if_exists); + bidir.source_origin_name = pg_strdup(bidir.source_slot_name); + pg_free(source_sub_name); + + /* + * Persist a pending-cleanup record now, before the base backup + * even starts: the source slot above already exists on the + * remote node, and a failed/interrupted backup would otherwise + * orphan it with nothing for --cleanup to find (the real + * manifest can't be written yet -- data_dir must stay empty for + * pg_basebackup). Superseded and removed once the real + * manifest is written below. + */ + bidir.manifest_path = bidir_pending_path; + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + bidir.manifest_path = bidir_manifest_path; + PQfinish(provider_conn); provider_conn = NULL; - exit(0); + break; /* single-database only, enforced above */ } /* only need to do this piece once */ @@ -1217,6 +2267,26 @@ main(int argc, char **argv) extra_basebackup_args); snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + /* + * Manifest write is deferred until here: pg_basebackup requires an + * empty target directory, and a manifest file in data_dir earlier + * would make it look non-empty. The pending-cleanup sidecar written + * right after source slot creation covers the gap between then and + * now; it's superseded by the real manifest and removed below. + */ + if (bidir.enabled) + { + write_manifest(&bidir, subscriber_name, database_list[0], base_prov_connstr); + if (unlink(bidir_pending_path) != 0 && errno != ENOENT) + print_msg(VERBOSITY_NORMAL, + _("warning: could not remove superseded pending sidecar " + "%s: %s\n"), bidir_pending_path, strerror(errno)); + print_msg(VERBOSITY_NORMAL, + _("Bidirectional plumbing complete: %d peer(s) discovered, " + "source slot created, manifest written to %s.\n"), + bidir.num_peers, bidir.manifest_path); + } + restore_point_name = generate_restore_point_name(); print_msg(VERBOSITY_NORMAL, _("Creating restore point \"%s\" on remote node ...\n"), @@ -1251,12 +2321,22 @@ main(int argc, char **argv) /* * Start subscriber node with spock disabled, and wait until it starts * accepting connections which means it has caught up to the restore point. + * + * TODO: for --bidirectional this node should be network-quarantined + * (private socket/listen address, or a restrictive pg_hba.conf) from + * this first startup through the end of the join -- spock.readonly = + * 'local' (set later) blocks writes but not reads or peer probes. Not + * implemented: --subscriber-dsn must be directly reachable, and the + * tool's own connections use that same DSN throughout, so restricting + * listen_addresses here would also lock the tool itself out. */ pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); if (pg_ctl_ret != 0) die(_("Postgres startup for restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); - wait_primary_connection(sub_connstr); + wait_primary_connection(sub_connstr, + bidir.enabled ? bidir.stall_timeout : 0, + bidir.enabled ? bidir.max_wait : 0); /* * Clean any per-node data that were copied by pg_basebackup. @@ -1264,30 +2344,122 @@ main(int argc, char **argv) print_msg(VERBOSITY_VERBOSE, _("Removing old spock configuration ...\n")); - for (dbnum = 0; dbnum < n_databases; dbnum++) + if (bidir.enabled) { - char *db = database_list[dbnum]; + Oid source_nodeid; + char *expected_sysid; + + /* + * Give n3 its own permanent identity now, right after promotion + * and before any catalog mutation: a physical backup preserves + * the source's system identifier, which risks stray WAL from one + * cluster being mistaken for the other's, and until reset makes + * system_identifier useless for proving a connection actually + * reaches n3 rather than the source. + */ + print_msg(VERBOSITY_NORMAL, + _("Assigning a new system identifier to the subscriber node...\n")); + pg_ctl_ret = run_pg_ctl("stop"); + if (pg_ctl_ret != 0) + die(_("Postgres stop before resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_shutdown(); - sub_connstr = get_connstr(base_sub_connstr, db); + { + sigset_t block_set, + old_set; + + /* + * Neither step below is safe to interrupt -- both write + * pg_control/WAL directly, and signal_handler() -> die() is + * not async-signal-safe. A signal landing mid-write could + * corrupt pg_control with no repair short of --cleanup + * --force. Block both signals across this pair of calls; + * any that arrives is deferred until right after. + */ + sigemptyset(&block_set); + sigaddset(&block_set, SIGINT); + sigaddset(&block_set, SIGTERM); + sigprocmask(SIG_BLOCK, &block_set, &old_set); + + expected_sysid = reset_subscriber_sysid(data_dir); + run_pg_resetwal(data_dir); + + sigprocmask(SIG_SETMASK, &old_set, NULL); + } - if (!sub_connstr || !strlen(sub_connstr)) - die(_("Subscriber connection string is not valid.\n")); + pg_ctl_ret = run_pg_ctl("start -l \"spock_create_subscriber_postgres.log\" -o \"-c shared_preload_libraries=''\""); + if (pg_ctl_ret != 0) + die(_("Postgres startup after resetting system identifier failed with %d."), pg_ctl_ret); + wait_postmaster_connection(sub_connstr); subscriber_conn = connectdb(sub_connstr); - remove_unwanted_data(subscriber_conn); + + /* + * --subscriber-dsn is expected to point directly at this node; + * verify that cheaply before running anything destructive, rather + * than trusting it silently. Now that n3 has just been given its + * own system identifier above, a straightforward comparison is a + * valid proof the connection reaches n3 and not the source or any + * other server -- unlike before the reset, nothing else could + * share it. + */ + { + PGresult *sysid_res = PQexec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + bool mismatch; + + if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) + { + PQclear(sysid_res); + die(_("could not verify --subscriber-dsn connects to this node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + mismatch = strcmp(PQgetvalue(sysid_res, 0, 0), expected_sysid) != 0; + PQclear(sysid_res); + if (mismatch) + die(_("--subscriber-dsn does not connect to the node at \"%s\": " + "system identifier mismatch. This can happen if the DSN " + "routes to the source node or another server; refusing " + "to run catalog operations against it.\n"), data_dir); + } + free(expected_sysid); + + /* Capture repset/table/sequence state before the catalog strip. */ + source_nodeid = get_local_node_id(subscriber_conn); + capture_catalog_state(subscriber_conn, source_nodeid, &capture); + + /* Drop all origins, then guarded DROP EXTENSION. */ + remove_unwanted_data_bidir(subscriber_conn, &capture); + PQfinish(subscriber_conn); subscriber_conn = NULL; } - - /* Stop Postgres so we can reset system id and start it with spock loaded. */ - pg_ctl_ret = run_pg_ctl("stop"); + else + { + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + + if (!sub_connstr || !strlen(sub_connstr)) + die(_("Subscriber connection string is not valid.\n")); + + subscriber_conn = connectdb(sub_connstr); + remove_unwanted_data(subscriber_conn); + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } + } + + /* Stop Postgres so we can start it again with spock (shared_preload_libraries) loaded. */ + pg_ctl_ret = run_pg_ctl("stop"); if (pg_ctl_ret != 0) die(_("Postgres stop after restore point catchup failed with %d. See spock_create_subscriber_postgres.log."), pg_ctl_ret); wait_postmaster_shutdown(); /* * Start the node again, now with spock active so that we can start the - * logical replication. This is final start, so don't log to to special log + * logical replication. This is final start, so don't log to to special log * file anymore. */ print_msg(VERBOSITY_NORMAL, @@ -1296,43 +2468,110 @@ main(int argc, char **argv) pg_ctl_ret = run_pg_ctl("start"); if (pg_ctl_ret != 0) die(_("Postgres restart with spock enabled failed with %d."), pg_ctl_ret); - wait_postmaster_connection(base_sub_connstr); + wait_postmaster_connection(bidir.enabled ? sub_connstr : base_sub_connstr); - for (dbnum = 0; dbnum < n_databases; dbnum++) + if (bidir.enabled) { - char *db = database_list[dbnum]; - - sub_connstr = get_connstr(base_sub_connstr, db); - prov_connstr = get_connstr(base_prov_connstr, db); + char *db = database_list[0]; subscriber_conn = connectdb(sub_connstr); - /* Create the extension. */ print_msg(VERBOSITY_VERBOSE, _("Creating spock extension for database %s...\n"), db); install_extension(subscriber_conn, "spock"); /* - * Create the identifier which is setup with the position to which we - * already caught up using physical replication. + * Create the local node, then immediately go read-only -- no + * window where n3 is reachable/writable before that lands. No + * origin creation here; the catchup subscription creates it + * later. + * + * dsn is --subscriber-dsn (sub_connstr) -- the externally-reachable + * address other nodes use to connect back, not a separate + * --node-dsn option. */ - print_msg(VERBOSITY_VERBOSE, - _("Creating replication origin for database %s...\n"), db); - initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), + subscriber_name); + { + PQExpBuffer nodequery = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(nodequery, + "SELECT spock.node_create(node_name := %s, dsn := %s)", + PQescapeLiteral(subscriber_conn, subscriber_name, + strlen(subscriber_name)), + PQescapeLiteral(subscriber_conn, sub_connstr, + strlen(sub_connstr))); + res = PQexec(subscriber_conn, nodequery->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create local node: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + destroyPQExpBuffer(nodequery); + } - /* - * And finally add the node to the cluster. - */ - print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), - subscriber_name, db); - print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + print_msg(VERBOSITY_NORMAL, _("Setting spock.readonly = 'local'...\n")); + set_readonly_local(subscriber_conn); + + /* Restore what was captured before the catalog strip. */ + print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + restore_replication_sets(subscriber_conn, &capture); - spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, - prov_connstr, replication_sets, apply_delay, - force_text_transfer); + bidir.source_restore_lsn = pg_strdup(remote_lsn); + bidir.node_dsn = sub_connstr; + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); PQfinish(subscriber_conn); subscriber_conn = NULL; + + print_msg(VERBOSITY_NORMAL, + _("Bidirectional join: physical backup, catalog strip, and " + "replication set restore complete. Node \"%s\" is " + "read-only pending the catchup subscription (a later " + "release).\n"), + subscriber_name); + } + else + { + for (dbnum = 0; dbnum < n_databases; dbnum++) + { + char *db = database_list[dbnum]; + + sub_connstr = get_connstr(base_sub_connstr, db); + prov_connstr = get_connstr(base_prov_connstr, db); + + subscriber_conn = connectdb(sub_connstr); + + /* Create the extension. */ + print_msg(VERBOSITY_VERBOSE, + _("Creating spock extension for database %s...\n"), db); + install_extension(subscriber_conn, "spock"); + + /* + * Create the identifier which is setup with the position to which we + * already caught up using physical replication. + */ + print_msg(VERBOSITY_VERBOSE, + _("Creating replication origin for database %s...\n"), db); + initialize_replication_origin(subscriber_conn, slot_names[dbnum], remote_lsn); + + /* + * And finally add the node to the cluster. + */ + print_msg(VERBOSITY_NORMAL, _("Creating subscriber %s for database %s...\n"), + subscriber_name, db); + print_msg(VERBOSITY_VERBOSE, _("Replication sets: %s\n"), replication_sets); + + spock_subscribe(subscriber_conn, subscriber_name, sub_connstr, + prov_connstr, replication_sets, apply_delay, + force_text_transfer); + + PQfinish(subscriber_conn); + subscriber_conn = NULL; + } } /* If user does not want the node to be running at the end, stop it. */ @@ -1367,7 +2606,10 @@ usage(void) printf(_(" pg_basebackup -X stream command\n")); printf(_(" --databases optional list of databases to replicate\n")); printf(_(" -n, --subscriber-name=NAME name of the newly created subscriber\n")); - printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber\n")); + printf(_(" --subscriber-dsn=CONNSTR connection string to the newly created subscriber;\n")); + printf(_(" with --bidirectional, also the externally-\n")); + printf(_(" reachable address peers use to connect back\n")); + printf(_(" to this node once joined (required)\n")); printf(_(" --provider-dsn=CONNSTR connection string to the provider\n")); printf(_(" --replication-sets=SETS comma separated list of replication set names\n")); printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); @@ -1383,6 +2625,19 @@ usage(void) printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); printf(_(" --recovery-conf path to the template recovery configuration\n")); + printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); + printf(_(" --bidirectional enable bidirectional join plumbing\n")); + printf(_(" --stall-timeout=SECS once PostgreSQL accepts connections, seconds of no\n")); + printf(_(" replay progress before giving up (default 600); does\n")); + printf(_(" not bound PostgreSQL's own startup\n")); + printf(_(" --max-wait=SECS hard ceiling on post-connection catchup wait, seconds\n")); + printf(_(" (default: unbounded); does not bound PostgreSQL's own\n")); + printf(_(" startup\n")); + printf(_(" --cleanup idempotently remove partial join state and exit\n")); + printf(_(" --force with --cleanup, also remove the data directory\n")); + printf(_("\nDuring the join, this node must be network-quarantined (private address /\n")); + printf(_("restrictive pg_hba.conf) by the operator -- via --hba-conf/--postgresql-conf --\n")); + printf(_("until the join completes; the tool does not manage this for you.\n")); } /* @@ -1432,7 +2687,7 @@ print_msg(VerbosityLevelEnum level, const char *fmt,...) /* * Start pg_ctl with given argument(s) - used to start/stop postgres * - * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a + * Returns the exit code reported by pg_ctl. If pg_ctl exits due to a * signal this call will die and not return. */ static int @@ -1442,7 +2697,7 @@ run_pg_ctl(const char *arg) PQExpBuffer cmd = createPQExpBuffer(); char *exec_path = find_other_exec_or_die(argv0, "pg_ctl"); - appendPQExpBuffer(cmd, "%s %s -D \"%s\"", exec_path, arg, data_dir); + appendPQExpBuffer(cmd, "\"%s\" %s -D \"%s\"", exec_path, arg, data_dir); /* Run pg_ctl in silent mode unless we run in debug mode. */ if (verbosity < VERBOSITY_DEBUG) @@ -1496,7 +2751,14 @@ run_basebackup(const char *provider_connstr, const char *data_dir, PQExpBuffer cmd = createPQExpBuffer(); char *exec_path = find_other_exec_or_die(argv0, "pg_basebackup"); - appendPQExpBuffer(cmd, "%s -D \"%s\" -d \"%s\" -X s -P", exec_path, data_dir, provider_connstr); + /* + * -c fast forces an immediate checkpoint. Without it, pg_basebackup + * requests the default "spread" checkpoint, which paces itself against + * checkpoint_timeout (5 minutes by default) regardless of how little + * data needs flushing -- an unpredictable, unnecessary stall for a + * tool whose entire job is this one backup. + */ + appendPQExpBuffer(cmd, "\"%s\" -D \"%s\" -d \"%s\" -X s -c fast -P", exec_path, data_dir, provider_connstr); /* Run pg_basebackup in verbose mode if we are running in verbose mode. */ if (verbosity >= VERBOSITY_VERBOSE) @@ -1643,144 +2905,832 @@ initialize_replication_slot(PGconn *conn, char *dbname, PQerrorMessage(conn)); } - PQclear(res); - resetPQExpBuffer(&query); + PQclear(res); + resetPQExpBuffer(&query); + + /* And finally, create the slot. */ + appendPQExpBuffer(&query, "SELECT pg_create_logical_replication_slot(%s, '%s');", + PQescapeLiteral(conn, slot_name, strlen(slot_name)), + "spock_output"); + + res = PQexec(conn, query.data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("Could not create replication slot, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + + PQclear(res); + termPQExpBuffer(&query); + + return slot_name; +} + +/* + * Read replication info about remote connection + * + * TODO: unify with spock_remote_node_info in spock_rpc + */ +static RemoteInfo * +get_remote_info(PGconn* conn) +{ + RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); + PGresult *res; + + if (!extension_exists(conn, "spock")) + die(_("The remote node is not configured as a spock provider.\n")); + + res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); + + /* No nodes found? */ + if (PQntuples(res) == 0) + die(_("The remote database is not configured as a spock node.\n")); + + if (PQntuples(res) > 1) + die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + +#define atooid(x) ((Oid) strtoul((x), NULL, 10)) + + ri->nodeid = atooid(PQgetvalue(res, 0, 0)); + ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); + ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); + ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); + ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + + PQclear(res); + + return ri; +} + +/* + * Check if extension exists. + */ +static bool +extension_exists(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + bool ret; + + printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", + PQescapeLiteral(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); + } + + ret = PQntuples(res) == 1; + + PQclear(res); + destroyPQExpBuffer(query); + + return ret; +} + +/* + * Create extension. + */ +static void +install_extension(PGconn *conn, const char *extname) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", + PQescapeIdentifier(conn, extname, strlen(extname))); + res = PQexec(conn, query->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + PQclear(res); + die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); + } + + PQclear(res); + destroyPQExpBuffer(query); +} + +/* + * Clean all the data that was copied from remote node but we don't + * want it here (currently shared security labels and replication identifiers). + */ +static void +remove_unwanted_data(PGconn *conn) +{ + PGresult *res; + + /* + * Remove replication identifiers (9.4 will get them removed by dropping + * the extension later as we emulate them there). + */ + res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + } + PQclear(res); + + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); +} + +/* + * Return the connected node's own local node id, from spock.local_node -- + * a plain catalog table, so it works even with spock's shared memory not + * loaded (e.g. before DROP EXTENSION, while spock is disabled). + */ +static Oid +get_local_node_id(PGconn *conn) +{ + PGresult *res; + Oid nodeid; + + res = PQexec(conn, "SELECT node_id FROM spock.local_node"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not determine source local node id: %s\n"), + PQerrorMessage(conn)); + } + nodeid = (Oid) strtoul(PQgetvalue(res, 0, 0), NULL, 10); + PQclear(res); + return nodeid; +} + +/* Capture replication set definitions owned by the source node. */ +static void +capture_repsets(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + + printfPQExpBuffer(query, + "SELECT set_name, replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set" + " WHERE set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replication set definitions: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_repsets = PQntuples(res); + capture->repsets = pg_malloc0(capture->num_repsets * sizeof(RepsetCapture)); + for (i = 0; i < capture->num_repsets; i++) + { + capture->repsets[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->repsets[i].replicate_insert = (PQgetvalue(res, i, 1)[0] == 't'); + capture->repsets[i].replicate_update = (PQgetvalue(res, i, 2)[0] == 't'); + capture->repsets[i].replicate_delete = (PQgetvalue(res, i, 3)[0] == 't'); + capture->repsets[i].replicate_truncate = (PQgetvalue(res, i, 4)[0] == 't'); + } + PQclear(res); + + destroyPQExpBuffer(query); +} + +/* Capture table memberships across all of the source's replication sets. */ +static void +capture_repset_tables(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + + printfPQExpBuffer(query, + "SELECT rs.set_name, rts.set_reloid::regclass AS qualified_table," + " rts.set_att_list AS columns," + " pg_get_expr(rts.set_row_filter, rts.set_reloid) AS row_filter" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replication set table memberships: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_tables = PQntuples(res); + capture->tables = pg_malloc0(capture->num_tables * sizeof(RepsetTableCapture)); + for (i = 0; i < capture->num_tables; i++) + { + capture->tables[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->tables[i].qualified_table = pg_strdup(PQgetvalue(res, i, 1)); + capture->tables[i].columns = PQgetisnull(res, i, 2) ? NULL : + pg_strdup(PQgetvalue(res, i, 2)); + capture->tables[i].row_filter = PQgetisnull(res, i, 3) ? NULL : + pg_strdup(PQgetvalue(res, i, 3)); + } + PQclear(res); + + destroyPQExpBuffer(query); +} + +/* + * Capture sequences and the sets they belong to (a sequence can be in more + * than one set, so this is captured per-membership like table rows, not + * deduplicated by sequence), plus each sequence's current value. + */ +static void +capture_repset_sequences(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + + printfPQExpBuffer(query, + "SELECT rs.set_name, rss.set_seqoid::regclass" + " FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_nodeid = %u", + source_nodeid); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not capture replicated sequence list: %s\n"), + PQerrorMessage(conn)); + } + + capture->num_sequences = PQntuples(res); + capture->sequences = pg_malloc0(capture->num_sequences * sizeof(SequenceCapture)); + for (i = 0; i < capture->num_sequences; i++) + { + PQExpBuffer seq_query = createPQExpBuffer(); + PGresult *seq_res; + + capture->sequences[i].set_name = pg_strdup(PQgetvalue(res, i, 0)); + capture->sequences[i].qualified_seq = pg_strdup(PQgetvalue(res, i, 1)); + + /* + * Read last_value/is_called directly off the sequence relation + * (standard technique) rather than pg_sequence_last_value(), which + * conflates "never called" with is_called=false and loses the + * distinction setval()'s third argument needs to restore exactly. + */ + printfPQExpBuffer(seq_query, "SELECT last_value, is_called FROM %s", + capture->sequences[i].qualified_seq); + seq_res = PQexec(conn, seq_query->data); + if (PQresultStatus(seq_res) != PGRES_TUPLES_OK) + { + PQclear(seq_res); + destroyPQExpBuffer(seq_query); + die(_("could not read sequence state for \"%s\": %s\n"), + capture->sequences[i].qualified_seq, PQerrorMessage(conn)); + } + + capture->sequences[i].last_value = strtoll(PQgetvalue(seq_res, 0, 0), NULL, 10); + capture->sequences[i].is_called = (PQgetvalue(seq_res, 0, 1)[0] == 't'); + + PQclear(seq_res); + destroyPQExpBuffer(seq_query); + } + PQclear(res); + + destroyPQExpBuffer(query); +} + +/* + * Capture replication-set definitions, table memberships, and sequence + * state from the local catalog before DROP EXTENSION removes it -- this + * reflects what was replicated at backup time, unlike querying the live + * source afterward. Utility-side memory only; never written to the + * manifest. + */ +static void +capture_catalog_state(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) +{ + memset(capture, 0, sizeof(*capture)); + + capture_repsets(conn, source_nodeid, capture); + capture_repset_tables(conn, source_nodeid, capture); + capture_repset_sequences(conn, source_nodeid, capture); + + print_msg(VERBOSITY_VERBOSE, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s) before catalog strip.\n"), + capture->num_repsets, capture->num_tables, capture->num_sequences); +} + +/* + * Bidirectional-mode catalog strip: drop ALL replication origins (not + * just ones with a status row, unlike remove_unwanted_data()), then + * guard DROP EXTENSION ... CASCADE with a one-hop pg_depend inventory -- + * any non-spock object depending on a spock member would otherwise be + * silently collaterally dropped. + */ +static void +remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) +{ + PGresult *res; + + (void) capture; /* must already be populated before this runs */ + + /* + * Drop all replication origins copied by the basebackup. + * pg_replication_origin is a cluster-wide (not per-database) catalog, + * so this is scoped to spock's own "spk_..." naming convention + * (gen_slot_name(), shared with slot names) rather than dropping every + * row -- an unrelated database on the same instance with its own + * (non-spock) logical replication would otherwise lose its origins too. + */ + res = PQexec(conn, + "SELECT pg_replication_origin_drop(roname)" + " FROM pg_replication_origin" + " WHERE roname LIKE 'spk\\_%' ESCAPE '\\'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not remove existing replication origins: %s\n"), + PQerrorMessage(conn)); + } + PQclear(res); + + /* Guard against CASCADE collaterally dropping user objects. */ + res = PQexec(conn, + "WITH spock_ext AS (" + " SELECT oid FROM pg_extension WHERE extname = 'spock'" + "), ext_members AS (" + " SELECT classid, objid FROM pg_depend, spock_ext" + " WHERE refclassid = 'pg_extension'::regclass" + " AND refobjid = spock_ext.oid" + " AND deptype = 'e'" + "), spock_members AS (" + /* + * Extension members proper (tables, views, functions, ...) + * plus anything with an INTERNAL ('i') or AUTO ('a') + * dependency on one of them -- a view's own rules use 'i', + * while a table's own constraints (CHECK, FK, ...) use + * 'a'; both are linked to their owning relation this way, + * not directly to the extension, but are just as much + * spock's own objects. + */ + " SELECT classid, objid FROM ext_members" + " UNION" + " SELECT d.classid, d.objid FROM pg_depend d" + " JOIN ext_members m ON d.refclassid = m.classid AND d.refobjid = m.objid" + " WHERE d.deptype IN ('i', 'a')" + ")" + "SELECT DISTINCT pg_describe_object(d.classid, d.objid, d.objsubid)" + " FROM pg_depend d" + " JOIN spock_members m ON d.refclassid = m.classid AND d.refobjid = m.objid" + " WHERE d.deptype = 'n'" + " AND NOT EXISTS (" + " SELECT 1 FROM spock_members m2" + " WHERE m2.classid = d.classid AND m2.objid = d.objid)"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not inventory spock extension dependents: %s\n"), + PQerrorMessage(conn)); + } + if (PQntuples(res) > 0) + { + PQExpBuffer list = createPQExpBuffer(); + int i; + + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "\n - %s", PQgetvalue(res, i, 0)); + + PQclear(res); + die(_("cannot drop the spock extension: the following object(s) " + "depend on it and would be collaterally dropped by CASCADE:%s\n" + "Resolve these dependencies manually before retrying; v1 does " + "not attempt to recreate them.\n"), + list->data); + } + PQclear(res); + + res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("Could not clean the spock extension, status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); +} + +/* + * Immediately after node_create, make the new node read-only to + * non-superuser clients -- there must be no window where n3 is + * reachable/writable before this lands. + */ +static void +set_readonly_local(PGconn *conn) +{ + PGresult *res; + + res = PQexec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("could not set spock.readonly: status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + res = PQexec(conn, "SELECT pg_reload_conf()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("could not reload configuration after setting spock.readonly: %s\n"), + PQerrorMessage(conn)); + } + PQclear(res); +} + +/* + * Restore the replication-set definitions, table memberships, and + * sequence state captured before the catalog strip, now that + * node_create() has given this node an identity again. Without this, + * n3 would accept incoming changes but send nothing back once peers + * create reverse subscriptions later. + */ +static void +restore_repsets(PGconn *conn, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + + for (i = 0; i < capture->num_repsets; i++) + { + RepsetCapture *s = &capture->repsets[i]; + bool builtin = (strcmp(s->set_name, "default") == 0 || + strcmp(s->set_name, "default_insert_only") == 0 || + strcmp(s->set_name, "ddl_sql") == 0); + + if (builtin) + printfPQExpBuffer(query, + "SELECT spock.repset_alter(" + "set_name := %s, " + "replicate_insert := %s, " + "replicate_update := %s, " + "replicate_delete := %s, " + "replicate_truncate := %s)", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), + s->replicate_insert ? "true" : "false", + s->replicate_update ? "true" : "false", + s->replicate_delete ? "true" : "false", + s->replicate_truncate ? "true" : "false"); + else + printfPQExpBuffer(query, + "SELECT spock.repset_create(" + "set_name := %s, " + "replicate_insert := %s, " + "replicate_update := %s, " + "replicate_delete := %s, " + "replicate_truncate := %s)", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), + s->replicate_insert ? "true" : "false", + s->replicate_update ? "true" : "false", + s->replicate_delete ? "true" : "false", + s->replicate_truncate ? "true" : "false"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not %s replication set \"%s\": %s\n"), + builtin ? "alter" : "recreate", s->set_name, PQerrorMessage(conn)); + } + PQclear(res); + } + + destroyPQExpBuffer(query); +} - /* And finally, create the slot. */ - appendPQExpBuffer(&query, "SELECT pg_create_logical_replication_slot(%s, '%s');", - PQescapeLiteral(conn, slot_name, strlen(slot_name)), - "spock_output"); +/* + * Restore table memberships for all sets. Named arguments are required: + * repset_add_table's 3rd positional argument is synchronize_data, not the + * column list, so a positional call would misfire. + * + * include_partitions := false: the capture already has a separate row per + * partition. Restoring the parent with include_partitions := true would + * re-add every child, violating the (set_id, set_reloid) primary key + * against the child's own captured row. + */ +static void +restore_repset_tables(PGconn *conn, CatalogCapture *capture) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; - res = PQexec(conn, query.data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) + for (i = 0; i < capture->num_tables; i++) { - die(_("Could not create replication slot, status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } + RepsetTableCapture *t = &capture->tables[i]; - PQclear(res); - termPQExpBuffer(&query); + printfPQExpBuffer(query, + "SELECT spock.repset_add_table(" + "set_name := %s, " + "relation := %s, " + "synchronize_data := false, " + "columns := %s, " + "row_filter := %s, " + "include_partitions := false)", + PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), + PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table)), + t->columns ? PQescapeLiteral(conn, t->columns, strlen(t->columns)) : "NULL", + t->row_filter ? PQescapeLiteral(conn, t->row_filter, strlen(t->row_filter)) : "NULL"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not add table \"%s\" to replication set \"%s\": %s\n"), + t->qualified_table, t->set_name, PQerrorMessage(conn)); + } + PQclear(res); + } - return slot_name; + destroyPQExpBuffer(query); } /* - * Read replication info about remote connection - * - * TODO: unify with spock_remote_node_info in spock_rpc + * Restore each sequence's replication-set membership, then its value, so + * n3 both publishes it and resumes it exactly. */ -static RemoteInfo * -get_remote_info(PGconn* conn) +static void +restore_repset_sequences(PGconn *conn, CatalogCapture *capture) { - RemoteInfo *ri = (RemoteInfo *)pg_malloc0(sizeof(RemoteInfo)); - PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; - if (!extension_exists(conn, "spock")) - die(_("The remote node is not configured as a spock provider.\n")); + for (i = 0; i < capture->num_sequences; i++) + { + SequenceCapture *sq = &capture->sequences[i]; - res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); + printfPQExpBuffer(query, + "SELECT spock.repset_add_seq(" + "set_name := %s, relation := %s, " + "synchronize_data := false)", + PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not add sequence \"%s\" to replication set \"%s\": %s\n"), + sq->qualified_seq, sq->set_name, PQerrorMessage(conn)); + } + PQclear(res); - /* No nodes found? */ - if (PQntuples(res) == 0) - die(_("The remote database is not configured as a spock node.\n")); + printfPQExpBuffer(query, "SELECT setval(%s, " INT64_FORMAT ", %s)", + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq)), + sq->last_value, + sq->is_called ? "true" : "false"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not restore sequence state for \"%s\": %s\n"), + sq->qualified_seq, PQerrorMessage(conn)); + } + PQclear(res); + } - if (PQntuples(res) > 1) - die(_("The remote database has multiple nodes configured. That is not supported with current version of spock.\n")); + destroyPQExpBuffer(query); +} -#define atooid(x) ((Oid) strtoul((x), NULL, 10)) +static void +restore_replication_sets(PGconn *conn, CatalogCapture *capture) +{ + /* + * 1. Recreate custom sets. The three built-in sets already exist from + * node_create(), so apply the captured flags to them via + * repset_alter() instead, since the source may have altered them. + */ + restore_repsets(conn, capture); - ri->nodeid = atooid(PQgetvalue(res, 0, 0)); - ri->node_name = pstrdup(PQgetvalue(res, 0, 1)); - ri->sysid = pstrdup(PQgetvalue(res, 0, 2)); - ri->dbname = pstrdup(PQgetvalue(res, 0, 3)); - ri->replication_sets = pstrdup(PQgetvalue(res, 0, 4)); + /* 2. Restore table memberships for all sets. */ + restore_repset_tables(conn, capture); - PQclear(res); + /* 3. Restore each sequence's replication-set membership and value. */ + restore_repset_sequences(conn, capture); - return ri; + print_msg(VERBOSITY_VERBOSE, + _("Restored %d replication set(s), %d table membership(s), " + "%d sequence(s).\n"), + capture->num_repsets, capture->num_tables, capture->num_sequences); + + verify_replication_sets_restored(conn, capture); } /* - * Check if extension exists. + * Verify the four replication-set flags landed correctly -- a bug in + * restore_repsets()'s argument binding would otherwise pass verification + * with wrong flags on the clone. */ -static bool -extension_exists(PGconn *conn, const char *extname) +static void +verify_repsets_restored(PGconn *conn, CatalogCapture *capture) { - PQExpBuffer query = createPQExpBuffer(); - PGresult *res; - bool ret; - - printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", - PQescapeLiteral(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; - if (PQresultStatus(res) != PGRES_TUPLES_OK) + for (i = 0; i < capture->num_repsets; i++) { + RepsetCapture *s = &capture->repsets[i]; + + printfPQExpBuffer(query, + "SELECT replicate_insert, replicate_update," + " replicate_delete, replicate_truncate" + " FROM spock.replication_set WHERE set_name = %s", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("replication set restore verification failed: set \"%s\" " + "not found after restore\n"), s->set_name); + } + + if (strcmp(PQgetvalue(res, 0, 0), s->replicate_insert ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 1), s->replicate_update ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 2), s->replicate_delete ? "t" : "f") != 0 || + strcmp(PQgetvalue(res, 0, 3), s->replicate_truncate ? "t" : "f") != 0) + { + char *got_insert = pg_strdup(PQgetvalue(res, 0, 0)); + char *got_update = pg_strdup(PQgetvalue(res, 0, 1)); + char *got_delete = pg_strdup(PQgetvalue(res, 0, 2)); + char *got_truncate = pg_strdup(PQgetvalue(res, 0, 3)); + + PQclear(res); + die(_("replication set restore verification failed: flags for set " + "\"%s\" do not match capture (expected i=%s/u=%s/d=%s/t=%s, " + "got i=%s/u=%s/d=%s/t=%s)\n"), + s->set_name, + s->replicate_insert ? "t" : "f", s->replicate_update ? "t" : "f", + s->replicate_delete ? "t" : "f", s->replicate_truncate ? "t" : "f", + got_insert, got_update, got_delete, got_truncate); + } PQclear(res); - die(_("Could not read extension info: %s\n"), PQerrorMessage(conn)); } - ret = PQntuples(res) == 1; - - PQclear(res); destroyPQExpBuffer(query); - - return ret; } /* - * Create extension. + * Verify table memberships: per-row column-list/row_filter round-trip, + * plus an aggregate COUNT(*) to catch an accidental double-add. n3 is + * brand-new here, so an unqualified COUNT(*) is safe. */ static void -install_extension(PGconn *conn, const char *extname) +verify_repset_tables_restored(PGconn *conn, CatalogCapture *capture) { - PQExpBuffer query = createPQExpBuffer(); - PGresult *res; - - printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", - PQescapeIdentifier(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + int count; - if (PQresultStatus(res) != PGRES_COMMAND_OK) + for (i = 0; i < capture->num_tables; i++) { + RepsetTableCapture *t = &capture->tables[i]; + char *columns; + char *row_filter; + + printfPQExpBuffer(query, + "SELECT rts.set_att_list," + " pg_get_expr(rts.set_row_filter, rts.set_reloid)" + " FROM spock.replication_set_table rts" + " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" + " WHERE rs.set_name = %s AND rts.set_reloid::regclass::text = %s", + PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), + PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("replication set restore verification failed: table \"%s\" " + "not found in set \"%s\" after restore\n"), + t->qualified_table, t->set_name); + } + + columns = PQgetisnull(res, 0, 0) ? NULL : pg_strdup(PQgetvalue(res, 0, 0)); + row_filter = PQgetisnull(res, 0, 1) ? NULL : pg_strdup(PQgetvalue(res, 0, 1)); PQclear(res); - die(_("Could not install %s extension: %s\n"), extname, PQerrorMessage(conn)); + + if ((columns == NULL) != (t->columns == NULL) || + (columns && strcmp(columns, t->columns) != 0)) + die(_("replication set restore verification failed: column list for " + "table \"%s\" in set \"%s\" does not match capture " + "(expected %s, got %s)\n"), + t->qualified_table, t->set_name, + t->columns ? t->columns : "NULL", columns ? columns : "NULL"); + + if ((row_filter == NULL) != (t->row_filter == NULL) || + (row_filter && strcmp(row_filter, t->row_filter) != 0)) + die(_("replication set restore verification failed: row_filter for " + "table \"%s\" in set \"%s\" does not re-parse identically " + "(expected %s, got %s)\n"), + t->qualified_table, t->set_name, + t->row_filter ? t->row_filter : "NULL", + row_filter ? row_filter : "NULL"); } + printfPQExpBuffer(query, + "SELECT COUNT(*) FROM spock.replication_set_table"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify table membership count: %s\n"), PQerrorMessage(conn)); + } + count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); + if (count != capture->num_tables) + die(_("replication set restore verification failed: expected %d table " + "membership(s), found %d\n"), capture->num_tables, count); + destroyPQExpBuffer(query); } /* - * Clean all the data that was copied from remote node but we don't - * want it here (currently shared security labels and replication identifiers). + * Verify sequence memberships: per-sequence membership, plus an aggregate + * COUNT(*) to catch an accidental double-add. */ static void -remove_unwanted_data(PGconn *conn) +verify_repset_sequences_restored(PGconn *conn, CatalogCapture *capture) { - PGresult *res; + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + int i; + int count; - /* - * Remove replication identifiers (9.4 will get them removed by dropping - * the extension later as we emulate them there). - */ - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) + for (i = 0; i < capture->num_sequences; i++) { + SequenceCapture *sq = &capture->sequences[i]; + + printfPQExpBuffer(query, + "SELECT COUNT(*) FROM spock.replication_set_seq rss" + " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" + " WHERE rs.set_name = %s AND rss.set_seqoid::regclass::text = %s", + PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), + PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify sequence membership for \"%s\": %s\n"), + sq->qualified_seq, PQerrorMessage(conn)); + } + count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); - die(_("Could not remove existing replication origins: %s\n"), PQerrorMessage(conn)); + if (count != 1) + die(_("replication set restore verification failed: sequence \"%s\" " + "not a member of set \"%s\" after restore\n"), + sq->qualified_seq, sq->set_name); } - PQclear(res); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); - if (PQresultStatus(res) != PGRES_COMMAND_OK) + printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_seq"); + res = PQexec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { - die(_("Could not clean the spock extension, status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + PQclear(res); + die(_("could not verify sequence membership count: %s\n"), PQerrorMessage(conn)); } + count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); + if (count != capture->num_sequences) + die(_("replication set restore verification failed: expected %d sequence " + "membership(s), found %d\n"), capture->num_sequences, count); + + destroyPQExpBuffer(query); +} + +/* + * Round-trip check: compare what actually landed against the capture, + * rather than trusting that each individual repset_add_table()/ + * repset_add_seq() call succeeding means the final state matches. + * Catches per-row drift (a column list or row_filter that didn't + * re-parse identically) and aggregate drift (an accidental double-add). + */ +static void +verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) +{ + verify_repsets_restored(conn, capture); + verify_repset_tables_restored(conn, capture); + verify_repset_sequences_restored(conn, capture); + + print_msg(VERBOSITY_VERBOSE, + _("Verified replication set restore matches capture exactly.\n")); } /* @@ -1800,7 +3750,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create replication origin \"%s\": status %s: %s\n"), - query->data, + origin_name, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } PQclear(res); @@ -1816,7 +3766,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not advance replication origin \"%s\": status %s: %s\n"), - query->data, + origin_name, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } PQclear(res); @@ -1837,7 +3787,8 @@ create_restore_point(PGconn *conn, char *restore_point_name) PGresult *res; char *remote_lsn = NULL; - printfPQExpBuffer(query, "SELECT pg_create_restore_point('%s')", restore_point_name); + printfPQExpBuffer(query, "SELECT pg_create_restore_point(%s)", + PQescapeLiteral(conn, restore_point_name, strlen(restore_point_name))); res = PQexec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -1937,7 +3888,7 @@ validate_replication_set_input(char *replication_sets) if (strlen(name) == 0) die(_("Replication set name \"%s\" is too short\n"), name); - if (strlen(name) > NAMEDATALEN) + if (strlen(name) >= NAMEDATALEN) die(_("Replication set name \"%s\" is too long\n"), name); for (cp = name; *cp; cp++) @@ -2090,25 +4041,97 @@ get_connstr(char *connstr, char *dbname) static char * read_sysid(const char *data_dir) { - ControlFileData ControlFile; - int fd; - char ControlFilePath[MAXPGPATH]; + ControlFileData *cf; + bool crc_ok; char *res = (char *) pg_malloc0(33); - snprintf(ControlFilePath, MAXPGPATH, "%s/global/pg_control", data_dir); + /* + * get_controlfile() validates the control file's CRC; a torn or + * corrupted control file must be rejected here rather than silently + * misread, since this result feeds directly into check_data_dir()'s + * "is this really a basebackup of the expected node" safety check. + */ + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); + + snprintf(res, 33, UINT64_FORMAT, cf->system_identifier); + pg_free(cf); + return res; +} + +/* + * Assign data_dir a fresh system identifier, since a physical clone + * otherwise keeps the source's -- risking stray WAL from one cluster + * being mistaken for the other's, and leaving system_identifier useless + * for proving --subscriber-dsn actually reaches this node. Called with + * the subscriber stopped, right after promotion and before any catalog + * mutation. + * + * pg_resetwal alone does NOT do this: it only regenerates + * system_identifier when it can't read an existing control file at all + * (verified empirically against a valid, cleanly-shut-down cluster). + * The identifier is overwritten directly in the control file here; + * pg_resetwal is run afterward (run_pg_resetwal()) only to relabel the + * existing WAL segments to match. + * + * Returns the new identifier as a string (caller must free()), matching + * read_sysid()'s convention. + */ +static char * +reset_subscriber_sysid(const char *data_dir) +{ + ControlFileData *cf; + bool crc_ok; + struct timeval tv; + char *result = (char *) pg_malloc0(33); - if ((fd = open(ControlFilePath, O_RDONLY | PG_BINARY, 0)) == -1) - die(_("%s: could not open file \"%s\" for reading: %s\n"), - progname, ControlFilePath, strerror(errno)); + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); - if (read(fd, &ControlFile, sizeof(ControlFileData)) != sizeof(ControlFileData)) - die(_("%s: could not read file \"%s\": %s\n"), - progname, ControlFilePath, strerror(errno)); + /* Same formula used to assign a system identifier at initdb time. */ + gettimeofday(&tv, NULL); + cf->system_identifier = ((uint64) tv.tv_sec) << 32; + cf->system_identifier |= ((uint64) tv.tv_usec) << 12; + cf->system_identifier |= getpid() & 0xFFF; - close(fd); + update_controlfile(data_dir, cf, true); - snprintf(res, 33, UINT64_FORMAT, ControlFile.system_identifier); - return res; + snprintf(result, 33, UINT64_FORMAT, cf->system_identifier); + pg_free(cf); + + return result; +} + +/* + * Relabel data_dir's existing WAL segments to match the system + * identifier reset_subscriber_sysid() just wrote to the control file + * (see that function's comment for why both steps are needed). Must + * run after reset_subscriber_sysid(), with the subscriber stopped. + */ +static void +run_pg_resetwal(const char *data_dir) +{ + int ret; + PQExpBuffer cmd = createPQExpBuffer(); + char *exec_path = find_other_exec_or_die(argv0, "pg_resetwal"); + + appendPQExpBuffer(cmd, "\"%s\" -D \"%s\"", exec_path, data_dir); + + print_msg(VERBOSITY_DEBUG, _("Running pg_resetwal: %s.\n"), cmd->data); + ret = system(cmd->data); + + destroyPQExpBuffer(cmd); + + if (WIFEXITED(ret) && WEXITSTATUS(ret) == 0) + return; + if (WIFEXITED(ret)) + die(_("pg_resetwal failed with exit status %d, cannot continue.\n"), WEXITSTATUS(ret)); + else if (WIFSIGNALED(ret)) + die(_("pg_resetwal exited with signal %d, cannot continue"), WTERMSIG(ret)); + else + die(_("pg_resetwal exited for an unknown reason (system() returned %d)"), ret); } /* @@ -2198,10 +4221,12 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) bool needquotes; /* - * If the string consists entirely of plain ASCII characters, no need to - * quote it. This is quite conservative, but better safe than sorry. + * If the string is one or more plain ASCII characters, no need to quote + * it. An empty string must default to needing quotes -- an unquoted + * empty value doesn't parse as empty, it swallows the entire next + * "keyword=value" token. */ - needquotes = false; + needquotes = true; for (s = str; *s; s++) { if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || @@ -2210,6 +4235,7 @@ appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str) needquotes = true; break; } + needquotes = false; } if (needquotes) @@ -2263,7 +4289,7 @@ wait_postmaster_connection(const char *connstr) break; /* - * Check if the process is still alive. This covers cases where the + * Check if the process is still alive. This covers cases where the * postmaster successfully created the pidfile but then crashed without * removing it. */ @@ -2280,14 +4306,25 @@ wait_postmaster_connection(const char *connstr) /* - * Wait for PostgreSQL to leave recovery/standby mode + * Wait for PostgreSQL to leave recovery/standby mode. + * + * stall_timeout/max_wait (seconds; 0 = disabled) bound replay catchup, + * but only once PostgreSQL first accepts connections -- they don't bound + * server startup itself. stall_timeout tracks pg_last_wal_replay_lsn() + * as a progress signal and fires only when replay stalls, not on total + * elapsed time, so a slow multi-GB catchup can still run. max_wait is a + * separate hard ceiling on total post-connection wait time. The + * unidirectional path passes 0/0 for unbounded waiting. */ static void -wait_primary_connection(const char *connstr) +wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) { bool ispri = false; PGconn *conn = NULL; PGresult *res; + time_t start_time = time(NULL); + time_t last_progress_time = start_time; + char *last_lsn = NULL; wait_postmaster_connection(connstr); @@ -2305,16 +4342,51 @@ wait_primary_connection(const char *connstr) res = PQexec(conn, "SELECT pg_is_in_recovery()"); if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') + { ispri = true; - else + PQclear(res); + break; + } + PQclear(res); + + if (stall_timeout > 0) { - pg_usleep(1000000); /* 1 sec */ - print_msg(VERBOSITY_VERBOSE, "."); + PGresult *lsn_res = PQexec(conn, "SELECT pg_last_wal_replay_lsn()"); + + if (PQresultStatus(lsn_res) == PGRES_TUPLES_OK && PQntuples(lsn_res) == 1 && + !PQgetisnull(lsn_res, 0, 0)) + { + char *cur_lsn = PQgetvalue(lsn_res, 0, 0); + + if (!last_lsn || strcmp(cur_lsn, last_lsn) != 0) + { + pg_free(last_lsn); + last_lsn = pg_strdup(cur_lsn); + last_progress_time = time(NULL); + } + } + PQclear(lsn_res); + + if ((time(NULL) - last_progress_time) >= stall_timeout) + { + PQfinish(conn); + die(_("recovery appears stalled: no WAL replay progress for " + "%d second(s) (--stall-timeout)\n"), stall_timeout); + } } - PQclear(res); + if (max_wait > 0 && (time(NULL) - start_time) >= max_wait) + { + PQfinish(conn); + die(_("timed out after %d second(s) waiting for recovery to " + "complete (--max-wait)\n"), max_wait); + } + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); } + pg_free(last_lsn); PQfinish(conn); print_msg(VERBOSITY_VERBOSE, "\n"); } @@ -2325,19 +4397,33 @@ wait_primary_connection(const char *connstr) static void wait_postmaster_shutdown(void) { - long pid; + long pid; + int waited = 0; + const int max_wait_secs = 60; print_msg(VERBOSITY_VERBOSE, "Waiting for PostgreSQL to shutdown ..."); for (;;) { - if ((pid = get_pgpid()) != 0) - { - pg_usleep(1000000); /* 1 sec */ - print_msg(VERBOSITY_NORMAL, "."); - } - else + pid = get_pgpid(); + if (pid == 0) + break; + + /* + * A hard-killed postmaster can leave its pidfile behind (it's only + * removed on a normal exit) -- without this check a stale pidfile + * hangs here forever. Mirrors the same postmaster_is_alive() check + * wait_postmaster_connection() already does on the start side. + */ + if (!postmaster_is_alive((pid_t) pid)) break; + + if (++waited >= max_wait_secs) + die(_("timed out after %d second(s) waiting for PostgreSQL to " + "shut down\n"), max_wait_secs); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_NORMAL, "."); } print_msg(VERBOSITY_VERBOSE, "\n"); @@ -2385,7 +4471,7 @@ copy_file(char *fromfile, char *tofile, bool append) #define COPY_BUF_SIZE (8 * BLCKSZ) - buffer = malloc(COPY_BUF_SIZE); + buffer = pg_malloc(COPY_BUF_SIZE); /* * Open the files @@ -2425,7 +4511,7 @@ copy_file(char *fromfile, char *tofile, bool append) /* we don't care about errors here */ close(srcfd); - free(buffer); + pg_free(buffer); } From 35dabb61dc581d23a624eae5f412737e93d1a561 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Wed, 5 Aug 2026 17:20:05 +0500 Subject: [PATCH 08/14] spock_create_subscriber: add --postgresql-auto-conf override pg_basebackup copies postgresql.auto.conf verbatim from the source, including any settings applied there via ALTER SYSTEM. Most of that (tuning, spock GUCs) is exactly what should carry over to the new node, but something like port may need to differ. postgresql.auto.conf is loaded after postgresql.conf and wins on conflicts, so such a setting can't be overridden via --postgresql-conf. Add --postgresql-auto-conf: its contents are appended to the inherited postgresql.auto.conf rather than replacing it, so only the settings it specifies override the source's, and everything else inherited stays in effect. --- .../spock_create_subscriber.c | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 96ed0ac18..a236fd0d3 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -198,8 +198,8 @@ static bool extension_exists(PGconn *conn, const char *extname); static void install_extension(PGconn *conn, const char *extname); static void initialize_data_dir(char *data_dir, char *connstr, - char *postgresql_conf, char *pg_hba_conf, - char *extra_basebackup_args); + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args); static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); static char *read_sysid(const char *data_dir); @@ -1832,6 +1832,7 @@ main(int argc, char **argv) char *replication_sets = NULL; char *databases = NULL; char *postgresql_conf = NULL, + *postgresql_auto_conf = NULL, *pg_hba_conf = NULL, *recovery_conf = NULL; int apply_delay = 0; @@ -1872,6 +1873,7 @@ main(int argc, char **argv) {"max-wait", required_argument, NULL, 14}, {"cleanup", no_argument, NULL, 15}, {"force", no_argument, NULL, 16}, + {"postgresql-auto-conf", required_argument, NULL, 17}, {NULL, 0, NULL, 0} }; @@ -1982,6 +1984,13 @@ main(int argc, char **argv) case 16: bidir.force_cleanup = true; break; + case 17: + { + postgresql_auto_conf = pg_strdup(optarg); + if (postgresql_auto_conf != NULL && !file_exists(postgresql_auto_conf)) + die(_("The specified postgresql.auto.conf file does not exist.")); + break; + } default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -2263,7 +2272,7 @@ main(int argc, char **argv) initialize_data_dir(data_dir, use_existing_data_dir ? NULL : prov_connstr, - postgresql_conf, pg_hba_conf, + postgresql_conf, postgresql_auto_conf, pg_hba_conf, extra_basebackup_args); snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); @@ -2624,6 +2633,7 @@ usage(void) printf(_("\nConfiguration files override:\n")); printf(_(" --hba-conf path to the new pg_hba.conf\n")); printf(_(" --postgresql-conf path to the new postgresql.conf\n")); + printf(_(" --postgresql-auto-conf settings to override in postgresql.auto.conf\n")); printf(_(" --recovery-conf path to the template recovery configuration\n")); printf(_("\nBidirectional join (joins an existing multi-master cluster):\n")); printf(_(" --bidirectional enable bidirectional join plumbing\n")); @@ -2793,8 +2803,8 @@ run_basebackup(const char *provider_connstr, const char *data_dir, */ static void initialize_data_dir(char *data_dir, char *connstr, - char *postgresql_conf, char *pg_hba_conf, - char *extra_basebackup_args) + char *postgresql_conf, char *postgresql_auto_conf, + char *pg_hba_conf, char *extra_basebackup_args) { if (connstr) { @@ -2805,6 +2815,32 @@ initialize_data_dir(char *data_dir, char *connstr, if (postgresql_conf) CopyConfFile(postgresql_conf, "postgresql.conf", false); + if (postgresql_auto_conf) + { + char auto_conf_path[MAXPGPATH]; + FILE *f; + + /* + * postgresql.auto.conf is copied verbatim from the source by + * pg_basebackup, and is loaded after postgresql.conf and wins + * on conflicts -- most of it (tuning, spock GUCs) is exactly + * what should carry over to this node, but a setting like port + * or listen_addresses may need to differ. Append rather than + * replace, so this node's overrides win (same-file, later + * setting wins) while everything else inherited stays in effect. + * A marker line makes any resulting duplicate settings obvious + * to whoever next reads the file. + */ + snprintf(auto_conf_path, sizeof(auto_conf_path), "%s/postgresql.auto.conf", data_dir); + f = fopen(auto_conf_path, "a"); + if (f == NULL) + die(_("could not open \"%s\": %s\n"), auto_conf_path, strerror(errno)); + fprintf(f, "# --- appended by spock_create_subscriber (--postgresql-auto-conf); " + "later settings override the inherited ones above ---\n"); + fclose(f); + + CopyConfFile(postgresql_auto_conf, "postgresql.auto.conf", true); + } if (pg_hba_conf) CopyConfFile(pg_hba_conf, "pg_hba.conf", false); } From 813330a8e813d3ee17ffab10bf7c6231e4519134 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 6 Aug 2026 19:19:42 +0500 Subject: [PATCH 09/14] spock_create_subscriber: trace every query and step at -v -v Add debug_exec(), a drop-in PQexec() wrapper that logs the query text and result status at VERBOSITY_DEBUG (-v -v), applied to every query site. Each major action also logs the concrete subscription, slot, node, or LSN it's acting on, so a -v -v run shows exactly what the tool is doing, not just which phase it's in. --- .../spock_create_subscriber.c | 192 ++++++++++++------ 1 file changed, 134 insertions(+), 58 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index a236fd0d3..1088d0592 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -166,6 +166,7 @@ static void die(const char *fmt,...) pg_attribute_printf(1, 2); static void print_msg(VerbosityLevelEnum level, const char *fmt,...) pg_attribute_printf(2, 3); +static PGresult *debug_exec(PGconn *conn, const char *query); static int run_pg_ctl(const char *arg); static void validate_extra_basebackup_args(const char *args); @@ -333,6 +334,7 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, int i; paramValues[0] = source_node_name; + print_msg(VERBOSITY_DEBUG, _(" > %s [$1=%s]\n"), discover_sql, source_node_name); res = PQexecParams(source_conn, discover_sql, 1, NULL, paramValues, NULL, NULL, 0); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -364,6 +366,10 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, paramValues[0] = dbname; paramValues[1] = peers[i].node_name; paramValues[2] = peers[i].sub_name; + print_msg(VERBOSITY_DEBUG, + _(" > SELECT spock.spock_gen_slot_name($1::name, $2::name, " + "$3::name) [$1=%s, $2=%s, $3=%s]\n"), + dbname, peers[i].node_name, peers[i].sub_name); slot_res = PQexecParams(source_conn, "SELECT spock.spock_gen_slot_name" "($1::name, $2::name, $3::name)", @@ -395,7 +401,7 @@ check_spock_version_at_least_6(PGconn *conn, const char *node_label) { PGresult *res; - res = PQexec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); + res = debug_exec(conn, "SELECT extversion FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -446,7 +452,7 @@ check_mesh_edges(PGconn *conn, const char *this_node_name, * means "actually replicating" -- a worker that's down or still * initializing must not satisfy the mesh. */ - res = PQexec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); + res = debug_exec(conn, "SELECT provider_node, status FROM spock.sub_show_status()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -525,7 +531,7 @@ check_peer_identity(PGconn *peer_conn, const char *expected_name) { PGresult *res; - res = PQexec(peer_conn, "SELECT node_name FROM spock.node_info()"); + res = debug_exec(peer_conn, "SELECT node_name FROM spock.node_info()"); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -576,7 +582,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " FROM spock.replication_set WHERE set_nodeid = %u" " AND (%s)" " ORDER BY set_name", node_id, selected_filter); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -610,7 +616,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " ORDER BY rts.set_reloid::regclass::text", node_id, PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); - tres = PQexec(conn, query->data); + tres = debug_exec(conn, query->data); if (PQresultStatus(tres) != PGRES_TUPLES_OK) { PQclear(tres); @@ -645,7 +651,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt "SELECT relkind::text, relreplident::text" " FROM pg_class WHERE oid = %s::regclass", PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); - cres = PQexec(conn, schema_query->data); + cres = debug_exec(conn, schema_query->data); if (PQresultStatus(cres) != PGRES_TUPLES_OK || PQntuples(cres) != 1) { PQclear(cres); @@ -670,7 +676,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " WHERE a.attrelid = %s::regclass AND a.attnum > 0" " AND NOT a.attisdropped ORDER BY a.attnum", PQescapeLiteral(conn, qualified_table, strlen(qualified_table))); - cres = PQexec(conn, schema_query->data); + cres = debug_exec(conn, schema_query->data); destroyPQExpBuffer(schema_query); if (PQresultStatus(cres) != PGRES_TUPLES_OK) { @@ -705,7 +711,7 @@ compute_repset_fingerprints(PGconn *conn, Oid node_id, const char *selected_filt " ORDER BY rss.set_seqoid::regclass::text", node_id, PQescapeLiteral(conn, entries[i].set_name, strlen(entries[i].set_name))); - sres = PQexec(conn, query->data); + sres = debug_exec(conn, query->data); if (PQresultStatus(sres) != PGRES_TUPLES_OK) { PQclear(sres); @@ -756,7 +762,7 @@ build_selected_set_name_filter(PGconn *conn) char *result; int i; - res = PQexec(conn, + res = debug_exec(conn, "SELECT DISTINCT s FROM spock.subscription," " unnest(sub_replication_sets) AS s ORDER BY 1"); if (PQresultStatus(res) != PGRES_TUPLES_OK) @@ -905,7 +911,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, check_spock_version_at_least_6(source_conn, "source"); /* track_commit_timestamp must be on at the source */ - res = PQexec(source_conn, "SHOW track_commit_timestamp"); + res = debug_exec(source_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("could not check track_commit_timestamp: %s"), PQerrorMessage(source_conn)); @@ -921,7 +927,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, * count is monotonically non-decreasing and is never zero on any node * that has replicated so much as a single DDL statement. */ - res = PQexec(source_conn, + res = debug_exec(source_conn, "SELECT COUNT(*) FROM pg_replication_slots" " WHERE slot_type = 'logical' AND plugin = 'spock_output'" " AND (confirmed_flush_lsn IS NULL" @@ -957,7 +963,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, check_peer_identity(peer_conn, peers[i].node_name); check_spock_version_at_least_6(peer_conn, peers[i].node_name); - res = PQexec(peer_conn, "SHOW track_commit_timestamp"); + res = debug_exec(peer_conn, "SHOW track_commit_timestamp"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { /* @@ -1012,7 +1018,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, PQExpBuffer others = createPQExpBuffer(); int other_count = 0; - res = PQexec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); + res = debug_exec(conn, "SELECT datname FROM pg_database WHERE NOT datistemplate"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -1043,7 +1049,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, "\"%s\" to check: %s\n"), dbname, errmsg); } - ext_res = PQexec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); + ext_res = debug_exec(db_conn, "SELECT 1 FROM pg_extension WHERE extname = 'spock'"); if (PQresultStatus(ext_res) != PGRES_TUPLES_OK) { char *errmsg = pg_strdup(PQerrorMessage(db_conn)); @@ -1058,7 +1064,7 @@ check_single_spock_database(PGconn *conn, const char *base_prov_connstr, if (PQntuples(ext_res) > 0) { - node_res = PQexec(db_conn, "SELECT 1 FROM spock.local_node"); + node_res = debug_exec(db_conn, "SELECT 1 FROM spock.local_node"); if (PQresultStatus(node_res) != PGRES_TUPLES_OK) { char *errmsg = pg_strdup(PQerrorMessage(db_conn)); @@ -1104,7 +1110,7 @@ check_no_native_subscriptions(PGconn *conn) { PGresult *res; - res = PQexec(conn, + res = debug_exec(conn, "SELECT s.subname, d.datname" " FROM pg_subscription s" " JOIN pg_database d ON d.oid = s.subdbid" @@ -1594,7 +1600,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " FROM pg_replication_slots" " WHERE slot_name = '%s'", state->source_slot_name); - res = PQexec(source_conn, query->data); + res = debug_exec(source_conn, query->data); if (PQresultStatus(res) == PGRES_TUPLES_OK) { if (PQntuples(res) > 0) @@ -1651,7 +1657,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, " FROM pg_replication_slots" " WHERE slot_name = '%s'", peer->slot_name); - res = PQexec(peer_conn, query->data); + res = debug_exec(peer_conn, query->data); if (PQresultStatus(res) == PGRES_TUPLES_OK) { if (PQntuples(res) > 0) @@ -1684,7 +1690,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", PQescapeLiteral(peer_conn, reverse_sub, strlen(reverse_sub))); - res = PQexec(peer_conn, query->data); + res = debug_exec(peer_conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { print_msg(VERBOSITY_NORMAL, @@ -2187,6 +2193,15 @@ main(int argc, char **argv) remote_info->node_name, subscriber_name, db, &bidir.peers); + { + int pi; + + for (pi = 0; pi < bidir.num_peers; pi++) + print_msg(VERBOSITY_DEBUG, + _("Discovered peer \"%s\" (dsn \"%s\", slot \"%s\")\n"), + bidir.peers[pi].node_name, bidir.peers[pi].dsn, + bidir.peers[pi].slot_name); + } check_preconditions(provider_conn, remote_info->node_name, bidir.peers, bidir.num_peers); check_single_spock_database(provider_conn, base_prov_connstr, db); @@ -2208,11 +2223,16 @@ main(int argc, char **argv) print_msg(VERBOSITY_NORMAL, _("Creating source replication slot in database %s ...\n"), db); + print_msg(VERBOSITY_DEBUG, + _("Creating replication slot on source \"%s\" for future " + "subscription \"%s\"\n"), remote_info->node_name, source_sub_name); bidir.source_slot_name = initialize_replication_slot(provider_conn, remote_info->dbname, remote_info->node_name, source_sub_name, drop_slot_if_exists); + print_msg(VERBOSITY_DEBUG, _("Source replication slot created: \"%s\"\n"), + bidir.source_slot_name); bidir.source_origin_name = pg_strdup(bidir.source_slot_name); pg_free(source_sub_name); @@ -2270,6 +2290,14 @@ main(int argc, char **argv) prov_connstr = get_connstr(base_prov_connstr, database_list[0]); sub_connstr = get_connstr(base_sub_connstr, database_list[0]); + if (!use_existing_data_dir) + print_msg(VERBOSITY_DEBUG, + _("Taking a physical base backup from \"%s\" into \"%s\"\n"), + prov_connstr, data_dir); + else + print_msg(VERBOSITY_DEBUG, + _("Reusing existing data directory \"%s\" (already a basebackup " + "of this source)\n"), data_dir); initialize_data_dir(data_dir, use_existing_data_dir ? NULL : prov_connstr, postgresql_conf, postgresql_auto_conf, pg_hba_conf, @@ -2413,7 +2441,7 @@ main(int argc, char **argv) * share it. */ { - PGresult *sysid_res = PQexec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); + PGresult *sysid_res = debug_exec(subscriber_conn, "SELECT system_identifier FROM pg_control_system()"); bool mismatch; if (PQresultStatus(sysid_res) != PGRES_TUPLES_OK || PQntuples(sysid_res) != 1) @@ -2434,9 +2462,20 @@ main(int argc, char **argv) /* Capture repset/table/sequence state before the catalog strip. */ source_nodeid = get_local_node_id(subscriber_conn); + print_msg(VERBOSITY_DEBUG, + _("Capturing replication-set/table/sequence membership for local " + "node id %u before dropping the spock extension\n"), source_nodeid); capture_catalog_state(subscriber_conn, source_nodeid, &capture); + print_msg(VERBOSITY_DEBUG, + _("Captured %d replication set(s), %d table membership(s), " + "%d sequence(s)\n"), + capture.num_repsets, capture.num_tables, capture.num_sequences); /* Drop all origins, then guarded DROP EXTENSION. */ + print_msg(VERBOSITY_DEBUG, + _("Dropping replication origins and the spock extension (checking " + "pg_depend first for non-spock objects CASCADE would collaterally " + "drop)\n")); remove_unwanted_data_bidir(subscriber_conn, &capture); PQfinish(subscriber_conn); @@ -2501,6 +2540,8 @@ main(int argc, char **argv) */ print_msg(VERBOSITY_NORMAL, _("Creating local Spock node \"%s\"...\n"), subscriber_name); + print_msg(VERBOSITY_DEBUG, _("Registering node \"%s\" with dsn \"%s\"\n"), + subscriber_name, sub_connstr); { PQExpBuffer nodequery = createPQExpBuffer(); PGresult *res; @@ -2511,7 +2552,7 @@ main(int argc, char **argv) strlen(subscriber_name)), PQescapeLiteral(subscriber_conn, sub_connstr, strlen(sub_connstr))); - res = PQexec(subscriber_conn, nodequery->data); + res = debug_exec(subscriber_conn, nodequery->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -2527,6 +2568,11 @@ main(int argc, char **argv) /* Restore what was captured before the catalog strip. */ print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); + print_msg(VERBOSITY_DEBUG, + _("Restoring %d replication set(s), %d table membership(s), " + "%d sequence(s) onto node \"%s\"\n"), + capture.num_repsets, capture.num_tables, capture.num_sequences, + subscriber_name); restore_replication_sets(subscriber_conn, &capture); bidir.source_restore_lsn = pg_strdup(remote_lsn); @@ -2624,7 +2670,9 @@ usage(void) printf(_(" --apply-delay=DELAY apply delay in seconds (by default 0)\n")); printf(_(" --drop-slot-if-exists drop replication slot of conflicting name\n")); printf(_(" -s, --stop stop the server once the initialization is done\n")); - printf(_(" -v increase logging verbosity\n")); + printf(_(" -v increase logging verbosity; repeatable --\n")); + printf(_(" -v -v also traces every query this tool\n")); + printf(_(" runs, with its result status\n")); printf(_(" --extra-basebackup-args additional arguments to pass to pg_basebackup.\n")); printf(_(" Safe options: -T, -c, --xlogdir/--waldir\n")); printf(_(" --text-types transfer column values as text rather than binary\n")); @@ -2693,6 +2741,34 @@ print_msg(VerbosityLevelEnum level, const char *fmt,...) } } +/* + * PQexec() wrapper that logs the query text at VERBOSITY_DEBUG (-v -v) + * before running it, and the resulting status/row count after -- a + * drop-in replacement so every query this tool issues is traceable + * without a separate print_msg() call at each site. Callers still do + * their own PQresultStatus()/die() handling on the result exactly as + * with a plain PQexec() call. + */ +static PGresult * +debug_exec(PGconn *conn, const char *query) +{ + PGresult *res; + + print_msg(VERBOSITY_DEBUG, _(" > %s\n"), query); + res = PQexec(conn, query); + if (verbosity >= VERBOSITY_DEBUG) + { + if (PQresultStatus(res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_DEBUG, _(" < %s (%d row(s))\n"), + PQresStatus(PQresultStatus(res)), PQntuples(res)); + else + print_msg(VERBOSITY_DEBUG, _(" < %s\n"), + PQresStatus(PQresultStatus(res))); + } + + return res; +} + /* * Start pg_ctl with given argument(s) - used to start/stop postgres @@ -2900,7 +2976,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, PQescapeLiteral(conn, subscription_name, strlen(subscription_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could generate slot name: %s"), PQerrorMessage(conn)); @@ -2914,7 +2990,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", PQescapeLiteral(conn, slot_name, strlen(slot_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could not fetch existing slot information: %s"), PQerrorMessage(conn)); @@ -2935,7 +3011,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, "SELECT pg_catalog.pg_drop_replication_slot(%s)", PQescapeLiteral(conn, slot_name, strlen(slot_name))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("Could not drop existing slot %s: %s"), slot_name, PQerrorMessage(conn)); @@ -2949,7 +3025,7 @@ initialize_replication_slot(PGconn *conn, char *dbname, PQescapeLiteral(conn, slot_name, strlen(slot_name)), "spock_output"); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create replication slot, status %s: %s\n"), @@ -2976,7 +3052,7 @@ get_remote_info(PGconn* conn) if (!extension_exists(conn, "spock")) die(_("The remote node is not configured as a spock provider.\n")); - res = PQexec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); + res = debug_exec(conn, "SELECT node_id, node_name, sysid, dbname, replication_sets FROM spock.node_info()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) die(_("could not fetch remote node info: %s\n"), PQerrorMessage(conn)); @@ -3012,7 +3088,7 @@ extension_exists(PGconn *conn, const char *extname) printfPQExpBuffer(query, "SELECT 1 FROM pg_catalog.pg_extension WHERE extname = %s;", PQescapeLiteral(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3039,7 +3115,7 @@ install_extension(PGconn *conn, const char *extname) printfPQExpBuffer(query, "CREATE EXTENSION IF NOT EXISTS %s;", PQescapeIdentifier(conn, extname, strlen(extname))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -3064,7 +3140,7 @@ remove_unwanted_data(PGconn *conn) * Remove replication identifiers (9.4 will get them removed by dropping * the extension later as we emulate them there). */ - res = PQexec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); + res = debug_exec(conn, "SELECT pg_replication_origin_drop(external_id) FROM pg_replication_origin_status;"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3072,7 +3148,7 @@ remove_unwanted_data(PGconn *conn) } PQclear(res); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + res = debug_exec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not clean the spock extension, status %s: %s\n"), @@ -3092,7 +3168,7 @@ get_local_node_id(PGconn *conn) PGresult *res; Oid nodeid; - res = PQexec(conn, "SELECT node_id FROM spock.local_node"); + res = debug_exec(conn, "SELECT node_id FROM spock.local_node"); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3118,7 +3194,7 @@ capture_repsets(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) " FROM spock.replication_set" " WHERE set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3157,7 +3233,7 @@ capture_repset_tables(PGconn *conn, Oid source_nodeid, CatalogCapture *capture) " JOIN spock.replication_set rs ON rts.set_id = rs.set_id" " WHERE rs.set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3199,7 +3275,7 @@ capture_repset_sequences(PGconn *conn, Oid source_nodeid, CatalogCapture *captur " JOIN spock.replication_set rs ON rss.set_id = rs.set_id" " WHERE rs.set_nodeid = %u", source_nodeid); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3225,7 +3301,7 @@ capture_repset_sequences(PGconn *conn, Oid source_nodeid, CatalogCapture *captur */ printfPQExpBuffer(seq_query, "SELECT last_value, is_called FROM %s", capture->sequences[i].qualified_seq); - seq_res = PQexec(conn, seq_query->data); + seq_res = debug_exec(conn, seq_query->data); if (PQresultStatus(seq_res) != PGRES_TUPLES_OK) { PQclear(seq_res); @@ -3289,7 +3365,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) * row -- an unrelated database on the same instance with its own * (non-spock) logical replication would otherwise lose its origins too. */ - res = PQexec(conn, + res = debug_exec(conn, "SELECT pg_replication_origin_drop(roname)" " FROM pg_replication_origin" " WHERE roname LIKE 'spk\\_%' ESCAPE '\\'"); @@ -3302,7 +3378,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) PQclear(res); /* Guard against CASCADE collaterally dropping user objects. */ - res = PQexec(conn, + res = debug_exec(conn, "WITH spock_ext AS (" " SELECT oid FROM pg_extension WHERE extname = 'spock'" "), ext_members AS (" @@ -3356,7 +3432,7 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) } PQclear(res); - res = PQexec(conn, "DROP EXTENSION spock CASCADE;"); + res = debug_exec(conn, "DROP EXTENSION spock CASCADE;"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("Could not clean the spock extension, status %s: %s\n"), @@ -3375,7 +3451,7 @@ set_readonly_local(PGconn *conn) { PGresult *res; - res = PQexec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); + res = debug_exec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { die(_("could not set spock.readonly: status %s: %s\n"), @@ -3383,7 +3459,7 @@ set_readonly_local(PGconn *conn) } PQclear(res); - res = PQexec(conn, "SELECT pg_reload_conf()"); + res = debug_exec(conn, "SELECT pg_reload_conf()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("could not reload configuration after setting spock.readonly: %s\n"), @@ -3439,7 +3515,7 @@ restore_repsets(PGconn *conn, CatalogCapture *capture) s->replicate_update ? "true" : "false", s->replicate_delete ? "true" : "false", s->replicate_truncate ? "true" : "false"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3485,7 +3561,7 @@ restore_repset_tables(PGconn *conn, CatalogCapture *capture) PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table)), t->columns ? PQescapeLiteral(conn, t->columns, strlen(t->columns)) : "NULL", t->row_filter ? PQescapeLiteral(conn, t->row_filter, strlen(t->row_filter)) : "NULL"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3519,7 +3595,7 @@ restore_repset_sequences(PGconn *conn, CatalogCapture *capture) "synchronize_data := false)", PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3532,7 +3608,7 @@ restore_repset_sequences(PGconn *conn, CatalogCapture *capture) PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq)), sq->last_value, sq->is_called ? "true" : "false"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3590,7 +3666,7 @@ verify_repsets_restored(PGconn *conn, CatalogCapture *capture) " replicate_delete, replicate_truncate" " FROM spock.replication_set WHERE set_name = %s", PQescapeLiteral(conn, s->set_name, strlen(s->set_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3650,7 +3726,7 @@ verify_repset_tables_restored(PGconn *conn, CatalogCapture *capture) " WHERE rs.set_name = %s AND rts.set_reloid::regclass::text = %s", PQescapeLiteral(conn, t->set_name, strlen(t->set_name)), PQescapeLiteral(conn, t->qualified_table, strlen(t->qualified_table))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) { PQclear(res); @@ -3683,7 +3759,7 @@ verify_repset_tables_restored(PGconn *conn, CatalogCapture *capture) printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_table"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3720,7 +3796,7 @@ verify_repset_sequences_restored(PGconn *conn, CatalogCapture *capture) " WHERE rs.set_name = %s AND rss.set_seqoid::regclass::text = %s", PQescapeLiteral(conn, sq->set_name, strlen(sq->set_name)), PQescapeLiteral(conn, sq->qualified_seq, strlen(sq->qualified_seq))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3736,7 +3812,7 @@ verify_repset_sequences_restored(PGconn *conn, CatalogCapture *capture) } printfPQExpBuffer(query, "SELECT COUNT(*) FROM spock.replication_set_seq"); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); @@ -3781,7 +3857,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) printfPQExpBuffer(query, "SELECT pg_replication_origin_create(%s)", PQescapeLiteral(conn, origin_name, strlen(origin_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3797,7 +3873,7 @@ initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn) PQescapeLiteral(conn, origin_name, strlen(origin_name)), remote_lsn); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -3825,7 +3901,7 @@ create_restore_point(PGconn *conn, char *restore_point_name) printfPQExpBuffer(query, "SELECT pg_create_restore_point(%s)", PQescapeLiteral(conn, restore_point_name, strlen(restore_point_name))); - res = PQexec(conn, query->data); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create restore point, status %s: %s\n"), @@ -3854,7 +3930,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, PQescapeLiteral(conn, subscriber_name, strlen(subscriber_name)), PQescapeLiteral(conn, subscriber_dsn, strlen(subscriber_dsn))); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create local node, status %s: %s\n"), @@ -3879,7 +3955,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, PQescapeLiteral(conn, repsets.data, repsets.len), apply_delay, (force_text_transfer ? "t" : "f")); - res = PQexec(conn, query.data); + res = debug_exec(conn, query.data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { die(_("Could not create subscription, status %s: %s\n"), @@ -3887,7 +3963,7 @@ spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, } PQclear(res); - res = PQexec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" + res = debug_exec(conn, "UPDATE spock.local_sync_status SET sync_status = 'r'" " WHERE sync_status != 'r'"); if (PQresultStatus(res) != PGRES_COMMAND_OK) { @@ -4376,7 +4452,7 @@ wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) conn = connectdb(connstr); } - res = PQexec(conn, "SELECT pg_is_in_recovery()"); + res = debug_exec(conn, "SELECT pg_is_in_recovery()"); if (PQresultStatus(res) == PGRES_TUPLES_OK && PQntuples(res) == 1 && *PQgetvalue(res, 0, 0) == 'f') { ispri = true; @@ -4387,7 +4463,7 @@ wait_primary_connection(const char *connstr, int stall_timeout, int max_wait) if (stall_timeout > 0) { - PGresult *lsn_res = PQexec(conn, "SELECT pg_last_wal_replay_lsn()"); + PGresult *lsn_res = debug_exec(conn, "SELECT pg_last_wal_replay_lsn()"); if (PQresultStatus(lsn_res) == PGRES_TUPLES_OK && PQntuples(lsn_res) == 1 && !PQgetisnull(lsn_res, 0, 0)) From 4f32ceea0cf305fcdb4bf367ce2a00c344875951 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 6 Aug 2026 19:48:19 +0500 Subject: [PATCH 10/14] spock_create_subscriber: catchup subscription phase Continue --bidirectional past replication-set restore into catchup: - Create n3's subscription to the source disabled-first (enabled := false), which sets it READY and creates its local replication origin atomically with no apply worker and no INIT window; advance that origin to the recorded restore LSN, confirm forward_origins landed as '{all}' (or forwarded peer changes would be silently dropped), then enable it. - Pre-create a disabled subscription to every peer, giving each its own local named origin the same way, without touching the peer at all -- no remote slot, no apply worker there yet. - Capture a catchup target via a single spock.sync_event() on the source. - Wait for the catchup subscription to reach that target: a progress watchdog (resets on any origin advance, not a flat wall-clock timeout, mirroring the existing WAL-replay wait), that aborts immediately if the subscription's own status reports disabled -- the signal an unresolvable apply exception leaves behind. --- tests/tap/schedule | 2 +- .../t/{048_bidir_pr3.pl => 048_bidir_join.pl} | 102 +- .../spock_create_subscriber.c | 871 ++++++++++++++++-- 3 files changed, 903 insertions(+), 72 deletions(-) rename tests/tap/t/{048_bidir_pr3.pl => 048_bidir_join.pl} (85%) diff --git a/tests/tap/schedule b/tests/tap/schedule index 3502d4916..f71dfd37c 100644 --- a/tests/tap/schedule +++ b/tests/tap/schedule @@ -58,7 +58,7 @@ test: 037_wire_format_datestyle test: 038_reserved_schema_ddl_guard test: 044_apply_change_logging test: 045_lsn_from_commit_ts -test: 048_bidir_pr3 +test: 048_bidir_join # Upgrade schema match test (builds from source, slow): #test: 018_upgrade_schema_match # diff --git a/tests/tap/t/048_bidir_pr3.pl b/tests/tap/t/048_bidir_join.pl similarity index 85% rename from tests/tap/t/048_bidir_pr3.pl rename to tests/tap/t/048_bidir_join.pl index a317e4515..09a152d22 100644 --- a/tests/tap/t/048_bidir_pr3.pl +++ b/tests/tap/t/048_bidir_join.pl @@ -1,11 +1,14 @@ #!/usr/bin/perl # ============================================================================= -# Test: 048_bidir_pr3.pl - spock_create_subscriber --bidirectional +# Test: 048_bidir_join.pl - spock_create_subscriber --bidirectional # ============================================================================= -# Validates the bidirectional node-join procedure: physical backup, recovery -# to a restore point, catalog strip (capture + origin drop + guarded DROP -# EXTENSION), and replication-set/table/sequence restore -- stopping short -# of the catchup subscription (a later step). +# Validates the bidirectional node-join procedure end to end: physical +# backup, recovery to a restore point, catalog strip (capture + origin drop +# + guarded DROP EXTENSION), replication-set/table/sequence restore, and +# catchup (disabled-first subscription to the source, disabled placeholder +# subscriptions to every peer, and a wait for n3 to reach a target LSN +# captured on the source) -- stopping short of enabling any direct peer +# subscription (a later step). # # Topology: # n1 <-> n2 (full bidirectional Spock subscriptions, existing 2-node @@ -25,10 +28,12 @@ # 1 sequence advanced past its initial value on n1 (setval fidelity check) # 1 partitioned table (parent + 2 children) added to custom set on n1 # 1 sequence with apostrophe in name added to custom set on n1 +# 1 peer-forwarding test table created on n1 +# 1 peer-forwarding test table replicated to n2 # 1 --bidirectional exits 0 # 1 n3 postgres is running # 1 spock extension installed cleanly on n3 (exactly one row) -# 1 n3 has no leftover replication origins from the basebackup +# 1 n3 has exactly the catchup and peer origins, none leftover from the basebackup # 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 # 1 spock.readonly is 'local' on n3 # 1 custom replication set restored on n3 with correct flags @@ -45,6 +50,11 @@ # 1 manifest: source_restore_lsn populated # 1 manifest: node_dsn populated # 1 source slot exists on n1 +# 1 catchup subscription sub_n3_n1 is replicating on n3 +# 1 disabled peer subscription sub_n3_n2 exists and is disabled +# 1 n3's origin for peer n2 starts at 0/0 before any post-join write +# 1 n2's post-join write reached n3 via forwarding through sub_n3_n1 +# 1 n3's origin for peer n2 advanced during catchup forwarding # 1 --cleanup --force exits 0 # 1 source slot removed from n1 after cleanup # 1 n3 data directory removed after cleanup --force @@ -69,12 +79,12 @@ # 1 pending sidecar removed once cleanup actually completed # 1 destroy_cluster # --- -# 57 total +# 64 total # ============================================================================= use strict; use warnings; -use Test::More tests => 57; +use Test::More tests => 64; use File::Path qw(remove_tree); use lib '.'; use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail @@ -183,6 +193,24 @@ q(SELECT setval('"weird''s_seq"', 7, true)); pass('sequence with apostrophe in name added to custom set on n1'); +# Table used later to verify n3's origin for peer n2 advances via forwarding. +# Created on n1 only and left to arrive on n2 via DDL replication (creating +# it directly on both sides races the already-established cross-wire DDL +# replay); spock.include_ddl_repset=on adds it to 'default' on each node +# once it lands there. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[0], '-d', $dbname, '-c', + "CREATE TABLE pr4_peer_tbl (id serial primary key, val text)"; +pass('peer-forwarding test table created on n1'); + +my $tbl_on_n2 = '0'; +for (1 .. 15) { + $tbl_on_n2 = scalar_query(2, + "SELECT COUNT(*) FROM pg_tables WHERE tablename = 'pr4_peer_tbl'"); + last if $tbl_on_n2 eq '1'; + sleep(1); +} +is($tbl_on_n2, '1', 'peer-forwarding test table replicated to n2'); + # check_preconditions() requires all of n1's outbound replication to have # caught up (no unreplicated DDL/data still in flight to n2); wait for the # setup above to drain. @@ -248,9 +276,14 @@ $ext_count =~ s/\s+//g; is($ext_count, '1', 'spock extension installed cleanly on n3 (exactly one row)'); +# By this point the catchup subscription and the one disabled peer +# subscription (n2) have each created their own origin -- exactly 2, not +# more. Anything beyond that would mean an origin survived from the +# basebackup instead of being dropped by the catalog strip. my $origin_count = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SELECT COUNT(*) FROM pg_replication_origin"`; $origin_count =~ s/\s+//g; -is($origin_count, '0', 'n3 has no leftover replication origins from the basebackup'); +is($origin_count, '2', + 'n3 has exactly the catchup and peer origins, none leftover from the basebackup'); my $n3_sysid = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT system_identifier FROM pg_control_system()"`; $n3_sysid =~ s/\s+//g; @@ -343,6 +376,57 @@ sub psql_capture { "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name LIKE 'spk_%n3%'"); ok($source_slot_exists >= 1, 'source slot exists on n1'); +# ============================================================================= +# TEST: catchup subscription created, enabled, and caught up; disabled peer +# subscription's origin advances via forwarding once n2 writes post-join. +# ============================================================================= +my $sub_status = ''; +for (1 .. 30) { + $sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n1')"); + last if $sub_status eq 'replicating'; + sleep(1); +} +is($sub_status, 'replicating', 'catchup subscription sub_n3_n1 is replicating on n3'); + +my $peer_sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n2')"); +is($peer_sub_status, 'disabled', 'disabled peer subscription sub_n3_n2 exists and is disabled'); + +# Origin name matches what create_disabled_peer_subscriptions() computed for +# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')). +my $n2_origin_name = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); + +my $n2_origin_query = + "SELECT COALESCE(s.remote_lsn::text, '0/0') FROM pg_replication_origin o " . + "LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id " . + "WHERE o.roname = '$n2_origin_name'"; + +my $n2_origin_lsn_initial = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +is($n2_origin_lsn_initial, '0/0', + "n3's origin for peer n2 starts at 0/0 before any post-join write"); + +# Write on n2 after the join; n1 forwards it to n3 via sub_n3_n1's +# forward_origins = '{all}', and maybe_advance_forwarded_origin() should move +# n3's origin for n2 off 0/0 even though the direct sub_n3_n2 stays disabled. +system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (val) VALUES ('from_n2_post_join')"; + +my $row_on_n3 = '0'; +for (1 .. 30) { + $row_on_n3 = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n2_post_join'"); + last if $row_on_n3 eq '1'; + sleep(1); +} +is($row_on_n3, '1', "n2's post-join write reached n3 via forwarding through sub_n3_n1"); + +my $n2_origin_lsn = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', $n2_origin_query); +isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced during catchup forwarding"); + # ============================================================================= # TEST: --cleanup --force removes source slot, data directory, and manifest # ============================================================================= diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 1088d0592..976c2d8b1 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,9 @@ typedef struct RemoteInfo { char *sysid; char *dbname; char *replication_sets; + TimeLineID timeline_id; /* current TLI, for detecting a data_dir + * left over from an already-promoted + * earlier attempt (see check_data_dir()) */ } RemoteInfo; typedef struct PeerNodeInfo @@ -97,6 +101,9 @@ typedef struct BidirectionalState char *node_dsn; /* DSN registered via spock.node_create(); * the address peers use to connect back to * this node. Derived from --subscriber-dsn. */ + char *node_sysid; /* n3's system_identifier; lets --cleanup + * confirm node_dsn still reaches this node + * before dropping subscriptions there. */ bool cleanup_mode; bool force_cleanup; /* --force: also remove the data directory * on --cleanup, not just remote state */ @@ -198,10 +205,12 @@ static RemoteInfo *get_remote_info(PGconn* conn); static bool extension_exists(PGconn *conn, const char *extname); static void install_extension(PGconn *conn, const char *extname); +static void ensure_trailing_newline(const char *path); static void initialize_data_dir(char *data_dir, char *connstr, char *postgresql_conf, char *postgresql_auto_conf, char *pg_hba_conf, char *extra_basebackup_args); static bool check_data_dir(char *data_dir, RemoteInfo *remoteinfo); +static void check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo); static char *read_sysid(const char *data_dir); @@ -214,6 +223,7 @@ static char *PQconninfoParamsToConnstr(const char *const * keywords, const char static void appendPQExpBufferConnstrValue(PQExpBuffer buf, const char *str); static bool file_exists(const char *path); +static char *expand_tilde(char *path); static bool is_pg_dir(const char *path); static void copy_file(char *fromfile, char *tofile, bool append); static char *find_other_exec_or_die(const char *argv0, const char *target); @@ -242,6 +252,9 @@ static bool read_manifest(const char *manifest_path, BidirectionalState *state, static bool cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn, bool force_rm_datadir); +static void stop_postgres_in_data_dir(void); +static bool remove_data_dir_if_forced(bool force); +static bool check_sysid_matches(PGconn *conn, const char *expected_sysid); static void append_json_string(PQExpBuffer buf, const char *str); static void check_single_spock_database(PGconn *conn, const char *base_prov_connstr, @@ -252,6 +265,15 @@ static void capture_catalog_state(PGconn *conn, Oid source_nodeid, static void remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture); static void restore_replication_sets(PGconn *conn, CatalogCapture *capture); static void verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture); +static void create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn); +static void create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets); +static char *get_catchup_target_lsn(const char *source_dsn); +static void wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait); static void set_readonly_local(PGconn *conn); static Oid get_local_node_id(PGconn *conn); @@ -323,7 +345,7 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, "SELECT DISTINCT n.node_name, ni.if_dsn" " FROM spock.subscription s" " JOIN spock.node n ON s.sub_origin = n.node_id" - " JOIN spock.node_interface ni ON n.node_id = ni.if_nodeid" + " JOIN spock.node_interface ni ON ni.if_id = s.sub_origin_if" " WHERE n.node_name != $1" " ORDER BY n.node_name"; const char *paramValues[3]; @@ -748,6 +770,50 @@ free_repset_fingerprints(RepsetFingerprintEntry *entries, int n) pg_free(entries); } +/* + * Return a comma-separated list of every replication set actually + * referenced by conn's own subscriptions (sub_replication_sets), rather + * than every set that happens to exist locally. A --bidirectional join + * uses this to make the joining node inherit the sets already in use by + * the cluster it's joining, instead of accepting a separately specified + * list that could diverge from what check_replication_set_equivalence() + * (just below, via the identical query) validates. Caller frees the + * result. + */ +static char * +get_source_mesh_replication_sets(PGconn *conn) +{ + PGresult *res; + PQExpBuffer list; + char *result; + int i; + + res = debug_exec(conn, + "SELECT DISTINCT s FROM spock.subscription," + " unnest(sub_replication_sets) AS s ORDER BY 1"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not determine the cluster's replication sets: %s\n"), + PQerrorMessage(conn)); + } + if (PQntuples(res) == 0) + { + PQclear(res); + die(_("no subscription references any replication set; cannot " + "determine which replication sets to use\n")); + } + + list = createPQExpBuffer(); + for (i = 0; i < PQntuples(res); i++) + appendPQExpBuffer(list, "%s%s", i > 0 ? "," : "", PQgetvalue(res, i, 0)); + PQclear(res); + + result = pg_strdup(list->data); + destroyPQExpBuffer(list); + return result; +} + /* * Build a SQL boolean expression ("set_name IN (...)") over the union of * every replication set actually referenced by conn's own subscriptions @@ -1193,6 +1259,11 @@ write_manifest(BidirectionalState *state, const char *subscriber_name, append_json_string(buf, state->node_dsn); appendPQExpBufferStr(buf, "\",\n"); + appendPQExpBufferStr(buf, " \"node_sysid\": \""); + if (state->node_sysid) + append_json_string(buf, state->node_sysid); + appendPQExpBufferStr(buf, "\",\n"); + appendPQExpBufferStr(buf, " \"peers\": [\n"); for (i = 0; i < state->num_peers; i++) { @@ -1455,6 +1526,8 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) s->bidir->source_restore_lsn = token; else if (strcmp(s->cur_field, "node_dsn") == 0) s->bidir->node_dsn = token; + else if (strcmp(s->cur_field, "node_sysid") == 0) + s->bidir->node_sysid = token; else pg_free(token); } @@ -1552,10 +1625,95 @@ read_manifest(const char *manifest_path, BidirectionalState *state, return true; } +/* + * If data_dir holds a running postmaster, stop it (fast mode) and wait + * for shutdown. No-op if data_dir is unset, doesn't exist, or has no + * postmaster.pid. + */ +static void +stop_postgres_in_data_dir(void) +{ + struct stat st; + + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return; + + snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); + if (stat(pid_file, &st) == 0) + { + print_msg(VERBOSITY_NORMAL, + _(" stopping postgres in %s ...\n"), data_dir); + run_pg_ctl("stop -m fast"); + wait_postmaster_shutdown(); + } +} + +/* + * If data_dir exists, remove it when force is true (stopping postgres in + * it first, defensively, in case the caller hasn't already); if force is + * false, leave it in place with a hint. Returns false only when removal + * was attempted and actually failed; a missing data_dir, an unset one, + * or force being false are all "nothing to report" and return true. + */ +static bool +remove_data_dir_if_forced(bool force) +{ + if (data_dir == NULL || !data_dir[0] || !file_exists(data_dir)) + return true; + + if (!force) + { + print_msg(VERBOSITY_NORMAL, + _(" data directory %s was left in place; pass --force " + "to remove it, or clean it up manually.\n"), data_dir); + return true; + } + + stop_postgres_in_data_dir(); + + print_msg(VERBOSITY_NORMAL, + _(" removing data directory %s ...\n"), data_dir); + if (!rmtree(data_dir, true)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not fully remove data directory " + "%s; remove it manually\n"), data_dir); + return false; + } + + return true; +} + +/* + * Check whether conn's system_identifier (from pg_control_system()) + * matches expected_sysid. Any failure to confirm -- query error, no + * row, or an outright mismatch -- returns false. + */ +static bool +check_sysid_matches(PGconn *conn, const char *expected_sysid) +{ + PGresult *res; + bool matches; + + res = debug_exec(conn, "SELECT system_identifier FROM pg_control_system()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + return false; + } + matches = strcmp(PQgetvalue(res, 0, 0), expected_sysid) == 0; + PQclear(res); + return matches; +} + /* * Idempotently remove bidirectional join state from all reachable nodes. - * Connects to the source and each peer; drops replication slots and - * reverse subscriptions created during a previous join attempt. + * Connects to the subscriber (n3) itself, the source, and each peer; + * drops n3's own catchup/disabled-peer subscriptions, replication slots, + * and reverse subscriptions created during a previous join attempt. + * spock.sub_drop() on n3 kills that subscription's local apply worker and + * drops the matching remote slot on its origin itself, so n3 never needs + * to be stopped just to release a slot it holds open elsewhere. * Connectivity and drop failures are logged as warnings, not fatal, so * cleanup attempts every remaining resource -- but each failure is * tracked, and the function returns true only if every recorded resource @@ -1569,6 +1727,7 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, bool force_rm_datadir) { PGconn *source_conn; + PGconn *n3_conn; PGresult *res; PQExpBuffer query = createPQExpBuffer(); int i; @@ -1577,6 +1736,94 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, print_msg(VERBOSITY_NORMAL, _("Cleaning up partial bidirectional join state ...\n")); + /* + * Drop any subscriptions this run created on n3 itself: the catchup + * subscription to the source and any disabled peer subscriptions. A + * freshly-provisioned n3 has no other legitimate spock.subscription + * rows, so it's safe to drop everything found -- but only once + * node_sysid confirms node_dsn still reaches that same n3, since a + * manifest can outlive the node it describes (DNS change, load + * balancer, reused port). node_dsn is only set once node_create() + * has run, so its absence just means there's nothing on n3 yet. + */ + if (state->node_dsn && state->node_dsn[0]) + { + n3_conn = PQconnectdb(state->node_dsn); + if (PQstatus(n3_conn) != CONNECTION_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to subscriber \"%s\"; its " + "subscription(s) may still exist: %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + PQfinish(n3_conn); + } + else if (!state->node_sysid || !state->node_sysid[0] || + !check_sysid_matches(n3_conn, state->node_sysid)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: node_dsn for subscriber \"%s\" cannot be " + "confirmed to still identify the node this run " + "created (missing or mismatched system identifier); " + "refusing to drop subscriptions there. Investigate " + "manually.\n"), subscriber_name); + fully_cleaned = false; + PQfinish(n3_conn); + } + else + { + res = debug_exec(n3_conn, "SELECT sub_name FROM spock.subscription"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + for (i = 0; i < PQntuples(res); i++) + { + char *sub_name = PQgetvalue(res, i, 0); + PGresult *drop_res; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(n3_conn, sub_name, strlen(sub_name))); + drop_res = debug_exec(n3_conn, query->data); + if (PQresultStatus(drop_res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_NORMAL, + _(" dropped subscriber subscription %s\n"), + sub_name); + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop subscriber " + "subscription %s: %s\n"), + sub_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(drop_res); + } + } + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not list subscriptions on " + "subscriber \"%s\": %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(res); + PQfinish(n3_conn); + } + } + + /* + * Stop n3's postmaster unconditionally (not gated by --force, which + * only governs removing the data directory). check_data_dir() and + * check_reused_data_dir_is_safe() explicitly support resuming a join + * into this same data_dir after a failed attempt, and that resume + * path (main(), the "start -l ..." pg_ctl call before catchup) assumes + * postgres is not already running here; leaving it up after + * `--cleanup` would make the very next retry fail outright. The + * subscription drops above already ran while n3 was still reachable, + * so this is just shutdown, not a substitute for them. + */ + stop_postgres_in_data_dir(); + source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) { @@ -1716,39 +1963,8 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, * The data directory a partial run may have created via basebackup. * Never touch it without --force. */ - if (data_dir != NULL && data_dir[0] && file_exists(data_dir)) - { - if (force_rm_datadir) - { - struct stat st; - - snprintf(pid_file, MAXPGPATH, "%s/postmaster.pid", data_dir); - if (stat(pid_file, &st) == 0) - { - print_msg(VERBOSITY_NORMAL, - _(" stopping postgres in %s before removing it ...\n"), - data_dir); - run_pg_ctl("stop -m fast"); - wait_postmaster_shutdown(); - } - - print_msg(VERBOSITY_NORMAL, - _(" removing data directory %s ...\n"), data_dir); - if (!rmtree(data_dir, true)) - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not fully remove data directory " - "%s; remove it manually\n"), data_dir); - fully_cleaned = false; - } - } - else - { - print_msg(VERBOSITY_NORMAL, - _(" data directory %s was left in place; pass --force " - "to remove it, or clean it up manually.\n"), data_dir); - } - } + if (!remove_data_dir_if_forced(force_rm_datadir)) + fully_cleaned = false; if (!fully_cleaned) { @@ -1909,7 +2125,7 @@ main(int argc, char **argv) switch (c) { case 'D': - data_dir = pg_strdup(optarg); + data_dir = expand_tilde(pg_strdup(optarg)); break; case 'n': subscriber_name = pg_strdup(optarg); @@ -1925,21 +2141,21 @@ main(int argc, char **argv) break; case 4: { - postgresql_conf = pg_strdup(optarg); + postgresql_conf = expand_tilde(pg_strdup(optarg)); if (postgresql_conf != NULL && !file_exists(postgresql_conf)) die(_("The specified postgresql.conf file does not exist.")); break; } case 5: { - pg_hba_conf = pg_strdup(optarg); + pg_hba_conf = expand_tilde(pg_strdup(optarg)); if (pg_hba_conf != NULL && !file_exists(pg_hba_conf)) die(_("The specified pg_hba.conf file does not exist.")); break; } case 6: { - recovery_conf = pg_strdup(optarg); + recovery_conf = expand_tilde(pg_strdup(optarg)); if (recovery_conf != NULL && !file_exists(recovery_conf)) die(_("The specified recovery configuration file does not exist.")); break; @@ -1992,7 +2208,7 @@ main(int argc, char **argv) break; case 17: { - postgresql_auto_conf = pg_strdup(optarg); + postgresql_auto_conf = expand_tilde(pg_strdup(optarg)); if (postgresql_auto_conf != NULL && !file_exists(postgresql_auto_conf)) die(_("The specified postgresql.auto.conf file does not exist.")); break; @@ -2043,7 +2259,19 @@ main(int argc, char **argv) if (apply_delay > MAX_APPLY_DELAY) die(_("Apply delay cannot be more than %d.\n"), MAX_APPLY_DELAY); - if (!replication_sets || !strlen(replication_sets)) + if (bidir.enabled) + { + /* + * n3 is joining an existing mesh, so its subscriptions must select + * exactly what the mesh already replicates; replication_sets is + * derived from the source's own subscriptions below instead. + */ + if (replication_sets != NULL) + die(_("--replication-sets cannot be combined with --bidirectional; " + "the joining node's replication sets are detected " + "automatically from the cluster it is joining.\n")); + } + else if (!replication_sets || !strlen(replication_sets)) replication_sets = "default,default_insert_only,ddl_sql"; /* Build the manifest path from --pgdata */ @@ -2086,6 +2314,25 @@ main(int argc, char **argv) exit(cleanup_partial_state(&bidir, sub_name, db, src_dsn, bidir.force_cleanup) ? 0 : 1); + /* + * Neither record exists -- there's no slot/subscription bookkeeping + * to act on, e.g. because the run died before the pending sidecar + * was even written. But an orphaned data_dir can still be sitting + * there from that attempt, and --force is an explicit instruction + * to remove it: don't leave it behind just because there was + * nothing to read. + */ + if (bidir.force_cleanup && data_dir != NULL && data_dir[0] && + file_exists(data_dir)) + { + fprintf(stderr, + _("No manifest found at %s or %s; no slot/subscription " + "state to clean up, but --force was given -- removing " + "data directory %s.\n"), + bidir.manifest_path, bidir_pending_path, data_dir); + exit(remove_data_dir_if_forced(true) ? 0 : 1); + } + fprintf(stderr, _("No manifest found at %s or %s; nothing to clean up.\n"), bidir.manifest_path, bidir_pending_path); exit(0); @@ -2189,6 +2436,17 @@ main(int argc, char **argv) PQExpBuffer sub_name_buf = createPQExpBuffer(); char *source_sub_name; + /* + * Inherit the replication sets already in use by the cluster + * being joined, rather than accept a separately specified + * list -- see the die() near the top of main() that rejects + * --replication-sets together with --bidirectional. + */ + replication_sets = get_source_mesh_replication_sets(provider_conn); + print_msg(VERBOSITY_VERBOSE, + _("Replication sets inherited from the existing " + "cluster: %s\n"), replication_sets); + bidir.num_peers = discover_peer_nodes(provider_conn, remote_info->node_name, subscriber_name, db, @@ -2208,13 +2466,7 @@ main(int argc, char **argv) check_no_native_subscriptions(provider_conn); use_existing_data_dir = check_data_dir(data_dir, remote_info); if (use_existing_data_dir) - { - char *local_sysid = read_sysid(data_dir); - bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; - free(local_sysid); - if (mismatch) - die(_("Subscriber data directory is not basebackup of remote node.\n")); - } + check_reused_data_dir_is_safe(data_dir, remote_info); appendPQExpBuffer(sub_name_buf, "sub_%s_%s", subscriber_name, remote_info->node_name); @@ -2261,13 +2513,7 @@ main(int argc, char **argv) use_existing_data_dir = check_data_dir(data_dir, remote_info); if (use_existing_data_dir) - { - char *local_sysid = read_sysid(data_dir); - bool mismatch = strcmp(remote_info->sysid, local_sysid) != 0; - free(local_sysid); - if (mismatch) - die(_("Subscriber data directory is not basebackup of remote node.\n")); - } + check_reused_data_dir_is_safe(data_dir, remote_info); } /* @@ -2458,7 +2704,13 @@ main(int argc, char **argv) "routes to the source node or another server; refusing " "to run catalog operations against it.\n"), data_dir); } - free(expected_sysid); + + /* + * Persist n3's own system identifier so --cleanup can re-verify + * node_dsn still reaches this same node later, rather than trusting + * a possibly stale manifest to still point at the right server. + */ + bidir.node_sysid = expected_sysid; /* Capture repset/table/sequence state before the catalog strip. */ source_nodeid = get_local_node_id(subscriber_conn); @@ -2579,14 +2831,63 @@ main(int argc, char **argv) bidir.node_dsn = sub_connstr; write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + { + PQExpBuffer sub_name_buf = createPQExpBuffer(); + char *source_sub_name; + char *target_lsn; + + appendPQExpBuffer(sub_name_buf, "sub_%s_%s", + subscriber_name, remote_info->node_name); + source_sub_name = pg_strdup(sub_name_buf->data); + destroyPQExpBuffer(sub_name_buf); + + print_msg(VERBOSITY_NORMAL, _("Creating catchup subscription to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Creating subscription \"%s\" to source \"%s\" using slot " + "\"%s\", forward_origins={all}, enabled=false\n"), + source_sub_name, prov_connstr, bidir.source_slot_name); + create_catchup_subscription(subscriber_conn, source_sub_name, prov_connstr, + replication_sets, bidir.source_slot_name, + bidir.source_restore_lsn); + print_msg(VERBOSITY_DEBUG, + _("Subscription \"%s\" created, origin advanced to %s, and " + "enabled\n"), source_sub_name, bidir.source_restore_lsn); + + print_msg(VERBOSITY_NORMAL, _("Creating disabled peer subscriptions...\n")); + create_disabled_peer_subscriptions(subscriber_conn, bidir.peers, + bidir.num_peers, replication_sets); + + /* + * Persist disabled_sub_created for every peer now, not just at + * the top of this block -- a crash during the (possibly long) + * catchup wait below must not leave --cleanup reading a stale + * manifest that still shows every peer's disabled subscription + * as not-yet-created. + */ + write_manifest(&bidir, subscriber_name, db, base_prov_connstr); + + print_msg(VERBOSITY_NORMAL, _("Getting catchup target from the source...\n")); + target_lsn = get_catchup_target_lsn(prov_connstr); + print_msg(VERBOSITY_DEBUG, _("Catchup target LSN: %s\n"), target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Waiting for catchup to the source...\n")); + print_msg(VERBOSITY_DEBUG, + _("Waiting for subscription \"%s\" (origin \"%s\") to reach " + "LSN %s\n"), source_sub_name, bidir.source_slot_name, target_lsn); + wait_for_catchup(subscriber_conn, source_sub_name, bidir.source_slot_name, + target_lsn, bidir.stall_timeout, bidir.max_wait); + + pg_free(target_lsn); + pg_free(source_sub_name); + } + PQfinish(subscriber_conn); subscriber_conn = NULL; print_msg(VERBOSITY_NORMAL, - _("Bidirectional join: physical backup, catalog strip, and " - "replication set restore complete. Node \"%s\" is " - "read-only pending the catchup subscription (a later " - "release).\n"), + _("Bidirectional join: catchup complete. Node \"%s\" has caught " + "up to the source and forward-tracked every peer's origin; " + "ready for the next phase.\n"), subscriber_name); } else @@ -2691,7 +2992,8 @@ usage(void) printf(_(" --max-wait=SECS hard ceiling on post-connection catchup wait, seconds\n")); printf(_(" (default: unbounded); does not bound PostgreSQL's own\n")); printf(_(" startup\n")); - printf(_(" --cleanup idempotently remove partial join state and exit\n")); + printf(_(" --cleanup idempotently remove partial join state and exit;\n")); + printf(_(" stops postgres if it is running in --pgdata\n")); printf(_(" --force with --cleanup, also remove the data directory\n")); printf(_("\nDuring the join, this node must be network-quarantined (private address /\n")); printf(_("restrictive pg_hba.conf) by the operator -- via --hba-conf/--postgresql-conf --\n")); @@ -2868,6 +3170,41 @@ run_basebackup(const char *provider_connstr, const char *data_dir, die(_("pg_basebackup exited for an unknown reason (system() returned %d)"), ret); } +/* + * Ensure path ends with a newline, appending one if it doesn't. Used + * after copying in a user-supplied config-file fragment that gets more + * content appended after it later (postgresql.auto.conf, where + * primary_conninfo is appended once recovery is configured) -- without + * this, a fragment file missing its own trailing newline would merge + * with whatever comes after it into one malformed setting. + */ +static void +ensure_trailing_newline(const char *path) +{ + int fd; + off_t size; + char last = '\0'; + + fd = open(path, O_RDWR | PG_BINARY); + if (fd < 0) + die(_("could not open \"%s\": %s\n"), path, strerror(errno)); + + size = lseek(fd, 0, SEEK_END); + if (size < 0) + die(_("could not seek in \"%s\": %s\n"), path, strerror(errno)); + + if (size > 0) + { + if (lseek(fd, -1, SEEK_CUR) < 0 || read(fd, &last, 1) != 1) + die(_("could not read \"%s\": %s\n"), path, strerror(errno)); + + if (last != '\n' && write(fd, "\n", 1) != 1) + die(_("could not write to \"%s\": %s\n"), path, strerror(errno)); + } + + close(fd); +} + /* * Init the datadir * @@ -2916,6 +3253,14 @@ initialize_data_dir(char *data_dir, char *connstr, fclose(f); CopyConfFile(postgresql_auto_conf, "postgresql.auto.conf", true); + + /* + * primary_conninfo is appended to this same file later, in + * WriteRecoveryConf(); if the override's last line lacks a + * trailing newline, that append would merge onto it instead of + * landing on its own line. + */ + ensure_trailing_newline(auto_conf_path); } if (pg_hba_conf) CopyConfFile(pg_hba_conf, "pg_hba.conf", false); @@ -2954,6 +3299,44 @@ check_data_dir(char *data_dir, RemoteInfo *remoteinfo) return false; } +/* + * Called whenever check_data_dir() approves reusing an existing + * data_dir. The sysid check alone doesn't catch every unsafe reuse: if + * an earlier attempt already reached promotion (recovery_target_action = + * promote) before failing or being interrupted, but before + * reset_subscriber_sysid() ran, the sysid still matches, yet this + * data_dir's own timeline has advanced past whatever the source has. + * Re-entering recovery against the source at that point can never + * succeed: the source has no way to supply WAL for a timeline it never + * had, so streaming fails permanently with "highest timeline N of the + * primary is behind recovery timeline M" and this tool would otherwise + * wait forever for WAL that will never arrive. Refuse reuse instead. + */ +static void +check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo) +{ + char *local_sysid = read_sysid(data_dir); + bool mismatch = strcmp(remoteinfo->sysid, local_sysid) != 0; + ControlFileData *cf; + bool crc_ok; + + free(local_sysid); + if (mismatch) + die(_("Subscriber data directory is not basebackup of remote node.\n")); + + cf = get_controlfile(data_dir, &crc_ok); + if (!crc_ok) + die(_("control file of \"%s\" appears to be corrupt\n"), data_dir); + if (cf->checkPointCopy.ThisTimeLineID > remoteinfo->timeline_id) + die(_("data directory \"%s\" is already on timeline %u, past the " + "source's current timeline %u -- it was already promoted by " + "an earlier, incomplete attempt and can never resume " + "recovery from this source again; run --cleanup --force and " + "retry with a fresh base backup\n"), + data_dir, cf->checkPointCopy.ThisTimeLineID, remoteinfo->timeline_id); + pg_free(cf); +} + /* * Initialize replication slots */ @@ -3073,6 +3456,12 @@ get_remote_info(PGconn* conn) PQclear(res); + res = debug_exec(conn, "SELECT timeline_id FROM pg_control_checkpoint()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + die(_("could not fetch remote node's current timeline: %s\n"), PQerrorMessage(conn)); + ri->timeline_id = (TimeLineID) strtoul(PQgetvalue(res, 0, 0), NULL, 10); + PQclear(res); + return ri; } @@ -3845,6 +4234,317 @@ verify_replication_sets_restored(PGconn *conn, CatalogCapture *capture) _("Verified replication set restore matches capture exactly.\n")); } +/* + * Create n3's subscription to the source disabled-first (enabled := + * false) -- sub_create() sets SYNC_STATUS_READY and creates the local + * replication origin atomically, with no apply worker and no INIT + * window, before this advances that origin to source_restore_lsn and + * enables it. + */ +static void +create_catchup_subscription(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_dsn, const char *replication_sets, + const char *source_slot_name, const char *source_restore_lsn) +{ + PQExpBuffer query = createPQExpBuffer(); + PQExpBuffer repsets = createPQExpBuffer(); + PGresult *res; + PGconn *source_conn; + + /* Re-confirm the source slot is still there before relying on it. */ + source_conn = connectdb(source_dsn); + printfPQExpBuffer(query, "SELECT 1 FROM pg_replication_slots WHERE slot_name = %s", + PQescapeLiteral(source_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(source_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check source replication slot \"%s\": %s\n"), + source_slot_name, PQerrorMessage(source_conn)); + } + if (PQntuples(res) != 1) + { + PQclear(res); + die(_("source replication slot \"%s\" is missing on the source; " + "cannot start catchup\n"), source_slot_name); + } + PQclear(res); + PQfinish(source_conn); + + printfPQExpBuffer(repsets, "{%s}", replication_sets); + printfPQExpBuffer(query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := false, " + "synchronize_data := false, " + "forward_origins := '{all}', " + "enabled := false)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name)), + PQescapeLiteral(subscriber_conn, source_dsn, strlen(source_dsn)), + PQescapeLiteral(subscriber_conn, repsets->data, repsets->len)); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create catchup subscription \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT pg_replication_origin_advance(%s, %s)", + PQescapeLiteral(subscriber_conn, source_slot_name, strlen(source_slot_name)), + PQescapeLiteral(subscriber_conn, source_restore_lsn, strlen(source_restore_lsn))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not advance catchup origin to the recovery point: %s\n"), + PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + /* + * Confirm forward_origins landed as '{all}' -- otherwise forwarded peer + * changes are silently dropped during catchup instead of reaching n3. + */ + printfPQExpBuffer(query, "SELECT forward_origins FROM spock.sub_show_status(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not verify forward_origins on \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + if (strcmp(PQgetvalue(res, 0, 0), "{all}") != 0) + { + char *got = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + die(_("catchup subscription \"%s\" has forward_origins = %s, expected " + "{all}; forwarded peer changes would be silently dropped\n"), + source_sub_name, got); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT spock.sub_enable(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not enable catchup subscription \"%s\": %s\n"), + source_sub_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + destroyPQExpBuffer(query); + destroyPQExpBuffer(repsets); +} + +/* + * Pre-create a disabled subscription to every peer on n3, giving each a + * local named origin (sub_create(enabled := false), same mechanism as + * the catchup subscription) without creating anything on the peer + * itself -- no remote slot, no apply worker. Forwarding through the + * catchup subscription is what advances these origins; the direct peer + * subscriptions stay disabled until a later phase. + */ +static void +create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, + int num_peers, const char *replication_sets) +{ + PQExpBuffer query = createPQExpBuffer(); + PQExpBuffer repsets = createPQExpBuffer(); + int i; + + printfPQExpBuffer(repsets, "{%s}", replication_sets); + + for (i = 0; i < num_peers; i++) + { + PeerNodeInfo *peer = &peers[i]; + PGresult *res; + + print_msg(VERBOSITY_DEBUG, + _("Creating disabled subscription \"%s\" to peer \"%s\" (dsn " + "\"%s\"); its origin will be \"%s\"\n"), + peer->sub_name, peer->node_name, peer->dsn, peer->slot_name); + printfPQExpBuffer(query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := false, " + "synchronize_data := false, " + "enabled := false)", + PQescapeLiteral(subscriber_conn, peer->sub_name, strlen(peer->sub_name)), + PQescapeLiteral(subscriber_conn, peer->dsn, strlen(peer->dsn)), + PQescapeLiteral(subscriber_conn, repsets->data, repsets->len)); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create disabled subscription \"%s\" to peer \"%s\": %s\n"), + peer->sub_name, peer->node_name, PQerrorMessage(subscriber_conn)); + } + PQclear(res); + + printfPQExpBuffer(query, "SELECT 1 FROM pg_replication_origin WHERE roname = %s", + PQescapeLiteral(subscriber_conn, peer->slot_name, strlen(peer->slot_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not verify replication origin for peer \"%s\": %s\n"), + peer->node_name, PQerrorMessage(subscriber_conn)); + } + if (PQntuples(res) != 1) + { + PQclear(res); + die(_("expected replication origin \"%s\" for peer \"%s\" was not " + "created\n"), peer->slot_name, peer->node_name); + } + PQclear(res); + + peer->disabled_sub_created = true; + } + + destroyPQExpBuffer(query); + destroyPQExpBuffer(repsets); +} + +/* + * A single spock.sync_event() on the source is the catchup target -- it + * flushes durably before returning, so the LSN is guaranteed to arrive + * at n3 via the replication stream with no per-peer flush needed. + * Caller must free the result. + */ +static char * +get_catchup_target_lsn(const char *source_dsn) +{ + PGconn *source_conn; + PGresult *res; + char *target_lsn; + + source_conn = connectdb(source_dsn); + res = debug_exec(source_conn, "SELECT spock.sync_event()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not get catchup target LSN: %s\n"), PQerrorMessage(source_conn)); + } + target_lsn = pg_strdup(PQgetvalue(res, 0, 0)); + PQclear(res); + PQfinish(source_conn); + + return target_lsn; +} + +/* + * Wait for n3's catchup subscription to reach target_lsn. Progress + * watchdog, not a flat wall-clock timeout -- reset the stall clock + * whenever remote_lsn advances at all, since a legitimately large + * catchup can take hours (same shape as wait_primary_connection(), which + * does this for WAL replay). Aborts immediately, without waiting out + * the timeout, if the subscription's own status reports 'disabled' -- + * the signal an unresolvable apply exception leaves behind under + * spock.exception_behaviour = 'sub_disable'; catchup must not be allowed + * to silently stall forever behind a stopped apply worker. + */ +static void +wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait) +{ + PQExpBuffer query = createPQExpBuffer(); + time_t start_time = time(NULL); + time_t last_progress_time = start_time; + char *last_lsn = NULL; + + print_msg(VERBOSITY_VERBOSE, "Waiting for catchup to reach %s...", target_lsn); + + for (;;) + { + PGresult *res; + bool reached; + + printfPQExpBuffer(query, + "SELECT (remote_lsn >= %s::pg_lsn), remote_lsn::text" + " FROM pg_replication_origin_status WHERE external_id = %s", + PQescapeLiteral(subscriber_conn, target_lsn, strlen(target_lsn)), + PQescapeLiteral(subscriber_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check catchup progress: %s\n"), PQerrorMessage(subscriber_conn)); + } + + reached = PQntuples(res) == 1 && !PQgetisnull(res, 0, 0) && + PQgetvalue(res, 0, 0)[0] == 't'; + if (reached) + { + PQclear(res); + break; + } + + if (PQntuples(res) == 1 && !PQgetisnull(res, 0, 1)) + { + char *cur_lsn = PQgetvalue(res, 0, 1); + + if (!last_lsn || strcmp(cur_lsn, last_lsn) != 0) + { + pg_free(last_lsn); + last_lsn = pg_strdup(cur_lsn); + last_progress_time = time(NULL); + } + } + PQclear(res); + + /* + * spock.sub_show_status() is the same primitive check_mesh_edges() + * relies on for subscription health; 'disabled' here means the + * apply worker hit an unresolvable exception and + * spock.exception_behaviour disabled it -- catchup cannot recover + * from that on its own, so abort now rather than waiting out + * stall_timeout/max_wait behind a subscription that will never + * move again. + */ + printfPQExpBuffer(query, "SELECT status FROM spock.sub_show_status(%s)", + PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(subscriber_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check catchup subscription status: %s\n"), + PQerrorMessage(subscriber_conn)); + } + if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "disabled") == 0) + { + PQclear(res); + die(_("catchup subscription \"%s\" was disabled during catchup, " + "likely by an unresolvable apply exception; this is a hard " + "join failure -- run --cleanup and retry\n"), source_sub_name); + } + PQclear(res); + + if (stall_timeout > 0 && (time(NULL) - last_progress_time) >= stall_timeout) + die(_("catchup appears stalled: no origin progress for %d second(s) " + "(--stall-timeout)\n"), stall_timeout); + + if (max_wait > 0 && (time(NULL) - start_time) >= max_wait) + die(_("timed out after %d second(s) waiting for catchup to " + "complete (--max-wait)\n"), max_wait); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + pg_free(last_lsn); + destroyPQExpBuffer(query); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + /* * Initialize new remote identifier to specific position. */ @@ -4552,6 +5252,53 @@ file_exists(const char *path) return true; } +/* + * Replace a leading "~" or "~username" with that user's home directory, + * in place. Shell tilde expansion never happens for a quoted argument, + * so a path like "~/n3.auto.conf" otherwise reaches file_exists() + * literally and fails. get_home_path() (port.h, already linked) + * resolves "~"/"~/..." for the current user; getpwnam() handles + * "~user"/"~user/...". + */ +static char * +expand_tilde(char *path) +{ + char *slash; + char home[MAXPGPATH]; + char *result; + + if (path == NULL || path[0] != '~') + return path; + + slash = strchr(path, '/'); + + if (slash == path + 1 || path[1] == '\0') + { + if (!get_home_path(home)) + return path; + } + else + { + char username[MAXPGPATH]; + struct passwd *pw; + size_t len = slash ? (size_t) (slash - (path + 1)) : strlen(path + 1); + + if (len >= sizeof(username)) + return path; + memcpy(username, path + 1, len); + username[len] = '\0'; + + pw = getpwnam(username); + if (pw == NULL) + return path; + strlcpy(home, pw->pw_dir, sizeof(home)); + } + + result = psprintf("%s%s", home, slash ? slash : ""); + pg_free(path); + return result; +} + static bool is_pg_dir(const char *path) { From 7536c1803af5cbe561d635661cdfc1cc4753b1dc Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 20 Aug 2026 18:12:03 +0500 Subject: [PATCH 11/14] spock_create_subscriber: bring n3 live with bidirectional replication Continue --bidirectional past catchup through cutover to a live, verified, writable bidirectional cluster member: - establish_peer_coverage_barrier(): a two-hop sync_event() barrier. For each peer, create its remote slot, then Hop 1 confirms the source has caught up to that peer (polled on the source, never n3 -- a peer's own marker is never forwarded). Once every peer's Hop 1 lands, Hop 2 confirms n3 has forward-coverage of all of them via one marker on the source. Re-checks replication-set/schema equivalence first, since the original precheck predates the (possibly long) catchup wait. - clear_forwarding(): clears forward_origins on the catchup subscription and waits for a provably new walsender connection (not just the catalog flip) before any direct peer subscription is enabled. - enable_peer_subs(): sub_enable() on each peer's disabled subscription now that forwarding is cleared. - create_reverse_subscriptions(): ordinary enabled sub_create() on every peer and the source, making replication bidirectional. A new source_reverse_sub_created flag lets --cleanup drop the source's reverse subscription without touching its other subscriptions. - wait_for_reverse_subs_ready() / verify_bidirectional_dataflow(): wait for the reverse subs, then prove data flows both ways using the barrier's own sync_event()/wait_for_origin_progress() mechanism. - lift_readonly(): lifts spock.readonly only after every subscription and both dataflow directions are verified. cleanup_partial_state() must stop n3's postmaster only after dropping the source's and each peer's reverse subscription, not before: those subscriptions use n3 as their provider, so spock.sub_drop() needs a live connection back to n3 to remove its provider-side slot -- Spock degrades that failure to a warning and still succeeds, so an earlier stop silently orphaned n3's slots under ordinary (non --force) cleanup. --- tests/tap/t/048_bidir_join.pl | 96 +- .../spock_create_subscriber.c | 1583 ++++++++++++++--- 2 files changed, 1375 insertions(+), 304 deletions(-) diff --git a/tests/tap/t/048_bidir_join.pl b/tests/tap/t/048_bidir_join.pl index 09a152d22..544eee36d 100644 --- a/tests/tap/t/048_bidir_join.pl +++ b/tests/tap/t/048_bidir_join.pl @@ -35,7 +35,7 @@ # 1 spock extension installed cleanly on n3 (exactly one row) # 1 n3 has exactly the catchup and peer origins, none leftover from the basebackup # 1 n3 was given its own system identifier (pg_resetwal), distinct from n1 -# 1 spock.readonly is 'local' on n3 +# 1 spock.readonly is lifted on n3 once the join is fully verified # 1 custom replication set restored on n3 with correct flags # 1 table membership restored with correct row_filter # 1 table membership restored with correct explicit column list @@ -51,10 +51,15 @@ # 1 manifest: node_dsn populated # 1 source slot exists on n1 # 1 catchup subscription sub_n3_n1 is replicating on n3 -# 1 disabled peer subscription sub_n3_n2 exists and is disabled -# 1 n3's origin for peer n2 starts at 0/0 before any post-join write -# 1 n2's post-join write reached n3 via forwarding through sub_n3_n1 -# 1 n3's origin for peer n2 advanced during catchup forwarding +# 1 forwarding cleared on sub_n3_n1 after cutover +# 1 direct peer subscription sub_n3_n2 is replicating on n3 after cutover +# 1 peer slot created on n2 during the coverage barrier +# 1 n2's post-cutover write reached n3 via the direct sub_n3_n2 path +# 1 n3's origin for peer n2 advanced via the direct subscription +# 1 reverse subscription sub_n2_n3 is replicating on n2 +# 1 reverse subscription sub_n1_n3 is replicating on n1 +# 1 n3's post-join write reached the source via sub_n1_n3 +# 1 n3's post-join write reached the peer via sub_n2_n3 # 1 --cleanup --force exits 0 # 1 source slot removed from n1 after cleanup # 1 n3 data directory removed after cleanup --force @@ -79,12 +84,12 @@ # 1 pending sidecar removed once cleanup actually completed # 1 destroy_cluster # --- -# 64 total +# 69 total # ============================================================================= use strict; use warnings; -use Test::More tests => 64; +use Test::More tests => 69; use File::Path qw(remove_tree); use lib '.'; use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail @@ -292,7 +297,8 @@ my $readonly = `$pg_bin/psql -p $n3_port -d $dbname -t -c "SHOW spock.readonly"`; $readonly =~ s/\s+//g; -is($readonly, 'local', "spock.readonly is 'local' on n3"); +is($readonly, 'off', + "spock.readonly is lifted on n3 once the join is fully verified"); my $repset_flags = `$pg_bin/psql -p $n3_port -d $dbname -t -A -c "SELECT replicate_insert, replicate_update, replicate_delete, replicate_truncate FROM spock.replication_set WHERE set_name = 'pr3_test_repset'"`; $repset_flags =~ s/\s+//g; @@ -377,8 +383,9 @@ sub psql_capture { ok($source_slot_exists >= 1, 'source slot exists on n1'); # ============================================================================= -# TEST: catchup subscription created, enabled, and caught up; disabled peer -# subscription's origin advances via forwarding once n2 writes post-join. +# TEST: catchup subscription and, after cutover, the direct peer +# subscription are both replicating; a post-cutover write on n2 reaches n3 +# via the direct path, advancing n3's origin for n2. # ============================================================================= my $sub_status = ''; for (1 .. 30) { @@ -389,28 +396,33 @@ sub psql_capture { } is($sub_status, 'replicating', 'catchup subscription sub_n3_n1 is replicating on n3'); +is(psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', + '-c', "SELECT forward_origins FROM spock.sub_show_status('sub_n3_n1')"), + '', "forwarding cleared on sub_n3_n1 after cutover"); + my $peer_sub_status = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', '-c', "SELECT status FROM spock.sub_show_status('sub_n3_n2')"); -is($peer_sub_status, 'disabled', 'disabled peer subscription sub_n3_n2 exists and is disabled'); +is($peer_sub_status, 'replicating', + 'direct peer subscription sub_n3_n2 is replicating on n3 after cutover'); # Origin name matches what create_disabled_peer_subscriptions() computed for -# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')). +# sub_n3_n2 (spock_gen_slot_name(dbname, 'n2', 'sub_n3_n2')) -- the same +# value is also the slot name create_peer_slot() created on n2 during the +# coverage barrier. my $n2_origin_name = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', '-c', "SELECT spock.spock_gen_slot_name('$dbname', 'n2', 'sub_n3_n2')"); +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT COUNT(*) FROM pg_replication_slots WHERE slot_name = '$n2_origin_name'"), + '1', 'peer slot created on n2 during the coverage barrier'); + my $n2_origin_query = "SELECT COALESCE(s.remote_lsn::text, '0/0') FROM pg_replication_origin o " . "LEFT JOIN pg_replication_origin_status s ON o.roident = s.local_id " . "WHERE o.roname = '$n2_origin_name'"; -my $n2_origin_lsn_initial = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', - '-c', $n2_origin_query); -is($n2_origin_lsn_initial, '0/0', - "n3's origin for peer n2 starts at 0/0 before any post-join write"); - -# Write on n2 after the join; n1 forwards it to n3 via sub_n3_n1's -# forward_origins = '{all}', and maybe_advance_forwarded_origin() should move -# n3's origin for n2 off 0/0 even though the direct sub_n3_n2 stays disabled. +# Write on n2 after cutover; forwarding is off and the direct sub_n3_n2 is +# enabled, so this reaches n3 directly from n2, not via n1. system_or_bail "$pg_bin/psql", '-p', $node_ports->[1], '-d', $dbname, '-c', "INSERT INTO pr4_peer_tbl (val) VALUES ('from_n2_post_join')"; @@ -421,11 +433,51 @@ sub psql_capture { last if $row_on_n3 eq '1'; sleep(1); } -is($row_on_n3, '1', "n2's post-join write reached n3 via forwarding through sub_n3_n1"); +is($row_on_n3, '1', "n2's post-cutover write reached n3 via the direct sub_n3_n2 path"); my $n2_origin_lsn = psql_capture('-p', $n3_port, '-d', $dbname, '-t', '-A', '-c', $n2_origin_query); -isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced during catchup forwarding"); +isnt($n2_origin_lsn, '0/0', "n3's origin for peer n2 advanced via the direct subscription"); + +# ============================================================================= +# TEST: reverse subscriptions are replicating, and a write on n3 reaches +# both the source and the peer through them -- an external proof, +# independent of the utility's own internal verify_bidirectional_dataflow() +# check. +# ============================================================================= +is(psql_capture('-p', $node_ports->[1], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n2_n3')"), + 'replicating', 'reverse subscription sub_n2_n3 is replicating on n2'); + +is(psql_capture('-p', $node_ports->[0], '-d', $dbname, '-t', '-A', + '-c', "SELECT status FROM spock.sub_show_status('sub_n1_n3')"), + 'replicating', 'reverse subscription sub_n1_n3 is replicating on n1'); + +# Explicit id: pr4_peer_tbl's serial sequence isn't part of the custom +# repset that gets its value round-tripped onto n3 (only pr3_test_seq and +# the apostrophe-named sequence are), so n3's own local copy of the +# sequence is still at its basebackup-time value and would collide with +# the id the n2-post-join row already claimed via replication. +system_or_bail "$pg_bin/psql", '-p', $n3_port, '-d', $dbname, '-c', + "INSERT INTO pr4_peer_tbl (id, val) VALUES (1000, 'from_n3_post_join')"; + +my $row_on_n1 = '0'; +for (1 .. 30) { + $row_on_n1 = scalar_query(1, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n1 eq '1'; + sleep(1); +} +is($row_on_n1, '1', "n3's post-join write reached the source via sub_n1_n3"); + +my $row_on_n2_from_n3 = '0'; +for (1 .. 30) { + $row_on_n2_from_n3 = scalar_query(2, + "SELECT COUNT(*) FROM pr4_peer_tbl WHERE val = 'from_n3_post_join'"); + last if $row_on_n2_from_n3 eq '1'; + sleep(1); +} +is($row_on_n2_from_n3, '1', "n3's post-join write reached the peer via sub_n2_n3"); # ============================================================================= # TEST: --cleanup --force removes source slot, data directory, and manifest diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 976c2d8b1..9fa3c1b5d 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -82,9 +82,18 @@ typedef struct PeerNodeInfo char *dsn; char *slot_name; /* from spock.spock_gen_slot_name() */ char *sub_name; /* "sub__" */ - bool disabled_sub_created; - bool slot_created; bool reverse_sub_created; + + /* + * slot_creation_lsn doubles as the "peer slot created" flag: a + * non-empty value means the slot exists, so no separate boolean is + * kept. It is persisted to the manifest for --cleanup; peer_marker_lsn + * below it is in-memory only -- it doesn't gate a --cleanup decision, + * and cutover has no cross-invocation resume in v1 (see + * check_reused_data_dir_is_safe()). + */ + char *slot_creation_lsn; /* peer's replication slot creation LSN */ + char *peer_marker_lsn; /* this peer's marker once seen on the source */ } PeerNodeInfo; typedef struct BidirectionalState @@ -95,15 +104,28 @@ typedef struct BidirectionalState int stall_timeout; /* default 600s */ int max_wait; /* default 0 = unbounded */ char *source_slot_name; - char *source_origin_name; char *source_restore_lsn; /* recovery target LSN; consumed by the - * disabled-first catchup sub_create */ + * disabled-first catchup sub_create. + * Written to the manifest as a + * diagnostic record of what was used, + * but never read back on a --cleanup + * reload -- nothing needs it restored + * into memory there */ char *node_dsn; /* DSN registered via spock.node_create(); * the address peers use to connect back to * this node. Derived from --subscriber-dsn. */ char *node_sysid; /* n3's system_identifier; lets --cleanup * confirm node_dsn still reaches this node * before dropping subscriptions there. */ + bool source_reverse_sub_created; /* sub__ created on + * the source -- must be + * persisted, unlike the peer + * progress bits, since it is + * the only way --cleanup knows + * to drop this specific + * subscription on a node that + * also has other, unrelated, + * legitimate subscriptions */ bool cleanup_mode; bool force_cleanup; /* --force: also remove the data directory * on --cleanup, not just remote state */ @@ -190,9 +212,14 @@ static char *validate_replication_set_input(char *replication_sets); static void remove_unwanted_data(PGconn *conn); static void initialize_replication_origin(PGconn *conn, char *origin_name, char *remote_lsn); static char *create_restore_point(PGconn *conn, char *restore_point_name); +static char *create_logical_slot(PGconn *conn, const char *slot_name, + bool drop_if_exists, const char *already_exists_hint); static char *initialize_replication_slot(PGconn *conn, char *dbname, char *provider_node_name, char *subscription_name, bool drop_slot_if_exists); +static char *create_peer_slot(PGconn *peer_conn, const char *peer_slot_name); +static char *get_origin_name_for_node(PGconn *conn, const char *upstream_node_name, + const char *conn_label, char **sub_name_out); static void spock_subscribe(PGconn *conn, char *subscriber_name, char *subscriber_dsn, char *provider_connstr, @@ -270,11 +297,48 @@ static void create_catchup_subscription(PGconn *subscriber_conn, const char *sou const char *source_slot_name, const char *source_restore_lsn); static void create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, int num_peers, const char *replication_sets); +static char *get_sync_event_lsn(PGconn *conn, const char *node_label); static char *get_catchup_target_lsn(const char *source_dsn); +static void wait_for_origin_progress(PGconn *conn, const char *origin_name, + const char *target_lsn, const char *watch_sub_name, + const char *context_label, int stall_timeout, int max_wait); static void wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, const char *source_slot_name, const char *target_lsn, int stall_timeout, int max_wait); +static void establish_peer_coverage_barrier(BidirectionalState *state, PGconn *n3_conn, + const char *prov_connstr, const char *source_node_name, + const char *source_sub_name, const char *source_slot_name, + const char *subscriber_name, const char *dbname, + const char *base_prov_connstr, + int stall_timeout, int max_wait); +static void clear_forwarding(PGconn *n3_conn, const char *source_dsn, + const char *source_sub_name, const char *source_slot_name, + int stall_timeout, int max_wait); +static void wait_for_sub_replicating(PGconn *conn, const char *sub_name, + int stall_timeout, int max_wait); +static void enable_peer_subs(PGconn *n3_conn, PeerNodeInfo *peers, int num_peers, + int stall_timeout, int max_wait); +static void create_subscription_on_conn(PGconn *conn, const char *sub_name, + const char *provider_dsn, const char *repsets, + const char *conn_label); +static void create_reverse_subscriptions(BidirectionalState *state, const char *subscriber_name, + const char *n3_dsn, const char *replication_sets, + const char *source_dsn, const char *source_node_name, + const char *dbname, const char *base_prov_connstr); +static void wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, int stall_timeout, int max_wait); +static void verify_dataflow_from_n3(PGconn *remote_conn, const char *remote_node_name, + const char *subscriber_name, const char *n3_marker, + int stall_timeout, int max_wait); +static void verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, + const char *remote_marker, const char *slot_name, + const char *sub_name, int stall_timeout, int max_wait); +static void verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, int stall_timeout, int max_wait); static void set_readonly_local(PGconn *conn); +static void lift_readonly(PGconn *conn); static Oid get_local_node_id(PGconn *conn); static PGconn * @@ -1209,160 +1273,178 @@ check_no_native_subscriptions(PGconn *conn) } /* - * Write the bidirectional state manifest to state->manifest_path - * atomically (write to .tmp, then rename). The manifest is a simple - * hand-formatted JSON file, with string values escaped by - * append_json_string(). + * Manifest field names, shared between serialize_manifest() and the + * manifest_* parser callbacks below, so a typo in one becomes a + * compile error instead of a silent read/write mismatch. + */ +#define MF_SUBSCRIBER_NAME "subscriber_name" +#define MF_DBNAME "dbname" +#define MF_SOURCE_DSN "source_dsn" +#define MF_SOURCE_SLOT_NAME "source_slot_name" +#define MF_SOURCE_RESTORE_LSN "source_restore_lsn" +#define MF_NODE_DSN "node_dsn" +#define MF_NODE_SYSID "node_sysid" +#define MF_SOURCE_REVERSE_SUB_CREATED "source_reverse_sub_created" +#define MF_PEERS "peers" +#define MF_NODE_NAME "node_name" +#define MF_PEER_DSN "peer_dsn" +#define MF_SUB_NAME_ON_N3 "sub_name_on_n3" +#define MF_PEER_SLOT_NAME "peer_slot_name" +#define MF_SLOT_CREATION_LSN "slot_creation_lsn" +#define MF_REVERSE_SUB_CREATED "reverse_sub_created" + +/* + * Append one """: """ field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). A NULL value serializes as an empty string, matching + * append_json_string()'s existing null-becomes-empty convention here. */ static void -write_manifest(BidirectionalState *state, const char *subscriber_name, - const char *dbname, const char *source_dsn) +append_json_str_field(PQExpBuffer buf, const char *indent, const char *field_name, + const char *value, bool trailing_comma) { - PQExpBuffer buf = createPQExpBuffer(); - char tmp_path[MAXPGPATH]; - int i; + appendPQExpBuffer(buf, "%s\"%s\": \"", indent, field_name); + if (value) + append_json_string(buf, value); + appendPQExpBufferStr(buf, trailing_comma ? "\",\n" : "\"\n"); +} + +/* + * Append one """: true|false" field to buf, with a + * trailing comma unless trailing_comma is false (the last field in an + * object). + */ +static void +append_json_bool_field(PQExpBuffer buf, const char *indent, const char *field_name, + bool value, bool trailing_comma) +{ + appendPQExpBuffer(buf, "%s\"%s\": %s%s\n", indent, field_name, + value ? "true" : "false", trailing_comma ? "," : ""); +} - snprintf(tmp_path, MAXPGPATH, "%s.tmp", state->manifest_path); +/* + * Hand-serialize the bidirectional state manifest schema into buf, as + * JSON, with string values escaped by append_json_string(). + */ +static void +serialize_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, PQExpBuffer buf) +{ + int i; appendPQExpBufferStr(buf, "{\n"); appendPQExpBufferStr(buf, " \"version\": 1,\n"); - appendPQExpBufferStr(buf, " \"subscriber_name\": \""); - append_json_string(buf, subscriber_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"dbname\": \""); - append_json_string(buf, dbname); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"source_dsn\": \""); - append_json_string(buf, source_dsn); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"source_slot_name\": \""); - if (state->source_slot_name) - append_json_string(buf, state->source_slot_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"source_origin_name\": \""); - if (state->source_origin_name) - append_json_string(buf, state->source_origin_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"source_restore_lsn\": \""); - if (state->source_restore_lsn) - append_json_string(buf, state->source_restore_lsn); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"node_dsn\": \""); - if (state->node_dsn) - append_json_string(buf, state->node_dsn); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"node_sysid\": \""); - if (state->node_sysid) - append_json_string(buf, state->node_sysid); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"peers\": [\n"); + append_json_str_field(buf, " ", MF_SUBSCRIBER_NAME, subscriber_name, true); + append_json_str_field(buf, " ", MF_DBNAME, dbname, true); + append_json_str_field(buf, " ", MF_SOURCE_DSN, source_dsn, true); + append_json_str_field(buf, " ", MF_SOURCE_SLOT_NAME, state->source_slot_name, true); + append_json_str_field(buf, " ", MF_SOURCE_RESTORE_LSN, state->source_restore_lsn, true); + append_json_str_field(buf, " ", MF_NODE_DSN, state->node_dsn, true); + append_json_str_field(buf, " ", MF_NODE_SYSID, state->node_sysid, true); + append_json_bool_field(buf, " ", MF_SOURCE_REVERSE_SUB_CREATED, + state->source_reverse_sub_created, true); + + appendPQExpBuffer(buf, " \"%s\": [\n", MF_PEERS); for (i = 0; i < state->num_peers; i++) { PeerNodeInfo *p = &state->peers[i]; bool last = (i == state->num_peers - 1); appendPQExpBufferStr(buf, " {\n"); - - appendPQExpBufferStr(buf, " \"node_name\": \""); - append_json_string(buf, p->node_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"peer_dsn\": \""); - append_json_string(buf, p->dsn); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"sub_name_on_n3\": \""); - append_json_string(buf, p->sub_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBufferStr(buf, " \"peer_slot_name\": \""); - append_json_string(buf, p->slot_name); - appendPQExpBufferStr(buf, "\",\n"); - - appendPQExpBuffer(buf, " \"disabled_sub_created\": %s,\n", - p->disabled_sub_created ? "true" : "false"); - appendPQExpBuffer(buf, " \"slot_created\": %s,\n", - p->slot_created ? "true" : "false"); - appendPQExpBuffer(buf, " \"reverse_sub_created\": %s\n", - p->reverse_sub_created ? "true" : "false"); - + append_json_str_field(buf, " ", MF_NODE_NAME, p->node_name, true); + append_json_str_field(buf, " ", MF_PEER_DSN, p->dsn, true); + append_json_str_field(buf, " ", MF_SUB_NAME_ON_N3, p->sub_name, true); + append_json_str_field(buf, " ", MF_PEER_SLOT_NAME, p->slot_name, true); + append_json_str_field(buf, " ", MF_SLOT_CREATION_LSN, p->slot_creation_lsn, true); + append_json_bool_field(buf, " ", MF_REVERSE_SUB_CREATED, + p->reverse_sub_created, false); appendPQExpBufferStr(buf, last ? " }\n" : " },\n"); } appendPQExpBufferStr(buf, " ]\n"); appendPQExpBufferStr(buf, "}\n"); +} - /* - * The manifest can embed a password (source_dsn, node_dsn), so create - * with mode 0600 up front, not a post-hoc chmod. O_EXCL|O_NOFOLLOW - * refuses to write through a pre-existing file or planted symlink, - * except a leftover .tmp from a previous crashed run. - */ - { - int fd; - ssize_t written; +/* + * Atomically replace path's contents with the len bytes at data: write + * to a ".tmp" sibling with mode 0600 up front (never a post-hoc + * chmod, since the caller's data can embed a password), fsync, rename + * over path, then fsync the containing directory -- a crash right + * after this returns must not lose the write (e.g. the only cleanup + * record for a resource created just before it). O_EXCL|O_NOFOLLOW + * refuses to write through a pre-existing file or planted symlink, + * except a leftover .tmp from a previous crashed run. + */ +static void +durably_replace_manifest(const char *path, const char *data, size_t len) +{ + char tmp_path[MAXPGPATH]; + int fd; + ssize_t written; - fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); - if (fd < 0 && errno == EEXIST) - { - if (unlink(tmp_path) != 0) - die(_("could not remove stale manifest temp file \"%s\": %s"), - tmp_path, strerror(errno)); - fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); - } - if (fd < 0) - die(_("could not create manifest file \"%s\": %s"), - tmp_path, strerror(errno)); + snprintf(tmp_path, MAXPGPATH, "%s.tmp", path); - written = write(fd, buf->data, buf->len); - if (written < 0 || (size_t) written != buf->len) - { - close(fd); - unlink(tmp_path); - die(_("could not write manifest file \"%s\": %s"), + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + if (fd < 0 && errno == EEXIST) + { + if (unlink(tmp_path) != 0) + die(_("could not remove stale manifest temp file \"%s\": %s"), tmp_path, strerror(errno)); - } + fd = open(tmp_path, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); + } + if (fd < 0) + die(_("could not create manifest file \"%s\": %s"), + tmp_path, strerror(errno)); - /* - * fsync, rename, then fsync the directory -- a crash right after - * this returns must not lose the only cleanup record for the - * source slot created just before it. - */ - if (fsync(fd) != 0) - { - close(fd); - unlink(tmp_path); - die(_("could not fsync manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - } - if (close(fd) != 0) - { - unlink(tmp_path); - die(_("could not close manifest file \"%s\": %s"), - tmp_path, strerror(errno)); - } - if (rename(tmp_path, state->manifest_path) != 0) - die(_("could not rename manifest to \"%s\": %s"), - state->manifest_path, strerror(errno)); + written = write(fd, data, len); + if (written < 0 || (size_t) written != len) + { + close(fd); + unlink(tmp_path); + die(_("could not write manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } - /* - * fsync_parent_path() already treats "filesystem doesn't support - * directory fsync" as success internally, so a nonzero return - * here is a genuine failure that can orphan the source slot - * after a crash. Fatal, like the durability steps above. - */ - if (fsync_parent_path(state->manifest_path) != 0) - die(_("could not fsync directory containing \"%s\": %s\n"), - state->manifest_path, strerror(errno)); + if (fsync(fd) != 0) + { + close(fd); + unlink(tmp_path); + die(_("could not fsync manifest file \"%s\": %s"), + tmp_path, strerror(errno)); } + if (close(fd) != 0) + { + unlink(tmp_path); + die(_("could not close manifest file \"%s\": %s"), + tmp_path, strerror(errno)); + } + if (rename(tmp_path, path) != 0) + die(_("could not rename manifest to \"%s\": %s"), + path, strerror(errno)); + + /* + * fsync_parent_path() already treats "filesystem doesn't support + * directory fsync" as success internally, so a nonzero return here + * is a genuine failure that can orphan the resource(s) this + * manifest is the only record of, after a crash. Fatal, like the + * durability steps above. + */ + if (fsync_parent_path(path) != 0) + die(_("could not fsync directory containing \"%s\": %s\n"), + path, strerror(errno)); +} + +/* + * Write the bidirectional state manifest to state->manifest_path. + */ +static void +write_manifest(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn) +{ + PQExpBuffer buf = createPQExpBuffer(); + + serialize_manifest(state, subscriber_name, dbname, source_dsn, buf); + durably_replace_manifest(state->manifest_path, buf->data, buf->len); destroyPQExpBuffer(buf); } @@ -1390,8 +1472,7 @@ typedef struct ManifestParseState char *peer_dsn; char *peer_sub_name; char *peer_slot_name; - bool peer_disabled_sub_created; - bool peer_slot_created; + char *peer_slot_creation_lsn; bool peer_reverse_sub_created; int peer_capacity; } ManifestParseState; @@ -1426,12 +1507,12 @@ manifest_object_end(void *st) s->bidir->peers[i].dsn = s->peer_dsn; s->bidir->peers[i].sub_name = s->peer_sub_name; s->bidir->peers[i].slot_name = s->peer_slot_name; - s->bidir->peers[i].disabled_sub_created = s->peer_disabled_sub_created; - s->bidir->peers[i].slot_created = s->peer_slot_created; + s->bidir->peers[i].slot_creation_lsn = s->peer_slot_creation_lsn; s->bidir->peers[i].reverse_sub_created = s->peer_reverse_sub_created; s->bidir->num_peers++; s->peer_node_name = s->peer_dsn = s->peer_sub_name = s->peer_slot_name = NULL; - s->peer_disabled_sub_created = s->peer_slot_created = s->peer_reverse_sub_created = false; + s->peer_slot_creation_lsn = NULL; + s->peer_reverse_sub_created = false; s->in_peer_obj = false; } s->depth--; @@ -1445,7 +1526,7 @@ manifest_array_start(void *st) s->depth++; if (s->depth == 2 && s->cur_field != NULL && - strcmp(s->cur_field, "peers") == 0) + strcmp(s->cur_field, MF_PEERS) == 0) s->in_peers = true; return JSON_SUCCESS; } @@ -1493,16 +1574,23 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) { bool value = (tokentype == JSON_TOKEN_TRUE); - if (strcmp(s->cur_field, "disabled_sub_created") == 0) - s->peer_disabled_sub_created = value; - else if (strcmp(s->cur_field, "slot_created") == 0) - s->peer_slot_created = value; - else if (strcmp(s->cur_field, "reverse_sub_created") == 0) + if (strcmp(s->cur_field, MF_REVERSE_SUB_CREATED) == 0) s->peer_reverse_sub_created = value; pg_free(token); return JSON_SUCCESS; } + /* Top-level creation-state flags are also JSON booleans. */ + if (!s->in_peer_obj && tokentype != JSON_TOKEN_STRING) + { + bool value = (tokentype == JSON_TOKEN_TRUE); + + if (strcmp(s->cur_field, MF_SOURCE_REVERSE_SUB_CREATED) == 0) + s->bidir->source_reverse_sub_created = value; + pg_free(token); + return JSON_SUCCESS; + } + if (tokentype != JSON_TOKEN_STRING) { pg_free(token); @@ -1512,21 +1600,17 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) if (!s->in_peer_obj) { /* top-level scalar fields */ - if (strcmp(s->cur_field, "subscriber_name") == 0) + if (strcmp(s->cur_field, MF_SUBSCRIBER_NAME) == 0) *s->subscriber_name_out = token; - else if (strcmp(s->cur_field, "dbname") == 0) + else if (strcmp(s->cur_field, MF_DBNAME) == 0) *s->dbname_out = token; - else if (strcmp(s->cur_field, "source_dsn") == 0) + else if (strcmp(s->cur_field, MF_SOURCE_DSN) == 0) *s->source_dsn_out = token; - else if (strcmp(s->cur_field, "source_slot_name") == 0) + else if (strcmp(s->cur_field, MF_SOURCE_SLOT_NAME) == 0) s->bidir->source_slot_name = token; - else if (strcmp(s->cur_field, "source_origin_name") == 0) - s->bidir->source_origin_name = token; - else if (strcmp(s->cur_field, "source_restore_lsn") == 0) - s->bidir->source_restore_lsn = token; - else if (strcmp(s->cur_field, "node_dsn") == 0) + else if (strcmp(s->cur_field, MF_NODE_DSN) == 0) s->bidir->node_dsn = token; - else if (strcmp(s->cur_field, "node_sysid") == 0) + else if (strcmp(s->cur_field, MF_NODE_SYSID) == 0) s->bidir->node_sysid = token; else pg_free(token); @@ -1534,14 +1618,16 @@ manifest_scalar(void *st, char *token, JsonTokenType tokentype) else { /* per-peer scalar fields */ - if (strcmp(s->cur_field, "node_name") == 0) + if (strcmp(s->cur_field, MF_NODE_NAME) == 0) s->peer_node_name = token; - else if (strcmp(s->cur_field, "peer_dsn") == 0) + else if (strcmp(s->cur_field, MF_PEER_DSN) == 0) s->peer_dsn = token; - else if (strcmp(s->cur_field, "sub_name_on_n3") == 0) + else if (strcmp(s->cur_field, MF_SUB_NAME_ON_N3) == 0) s->peer_sub_name = token; - else if (strcmp(s->cur_field, "peer_slot_name") == 0) + else if (strcmp(s->cur_field, MF_PEER_SLOT_NAME) == 0) s->peer_slot_name = token; + else if (strcmp(s->cur_field, MF_SLOT_CREATION_LSN) == 0) + s->peer_slot_creation_lsn = token; else pg_free(token); } @@ -1811,19 +1897,6 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, } } - /* - * Stop n3's postmaster unconditionally (not gated by --force, which - * only governs removing the data directory). check_data_dir() and - * check_reused_data_dir_is_safe() explicitly support resuming a join - * into this same data_dir after a failed attempt, and that resume - * path (main(), the "start -l ..." pg_ctl call before catchup) assumes - * postgres is not already running here; leaving it up after - * `--cleanup` would make the very next retry fail outright. The - * subscription drops above already ran while n3 was still reachable, - * so this is just shutdown, not a substitute for them. - */ - stop_postgres_in_data_dir(); - source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) { @@ -1865,12 +1938,60 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, PQclear(res); } + /* + * Drop the reverse subscription on the source if this run recorded + * having created it. Unlike n3 (whose n3-side block above + * drops every subscription it finds, since a fresh n3 has no other + * legitimate ones), the source has its own pre-existing, unrelated + * subscriptions that must not be touched -- so this is gated by the + * flag and targets the specific reverse subscription by name. + */ + if (source_conn && state->source_reverse_sub_created) + { + res = debug_exec(source_conn, "SELECT node_name FROM spock.node_info()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + print_msg(VERBOSITY_NORMAL, + _("warning: could not determine the source's node name; " + "its reverse subscription may still exist: %s\n"), + PQerrorMessage(source_conn)); + fully_cleaned = false; + } + else + { + char *source_node_name = pg_strdup(PQgetvalue(res, 0, 0)); + char reverse_sub[NAMEDATALEN * 2 + 8]; + + PQclear(res); + snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", + source_node_name, subscriber_name); + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(source_conn, reverse_sub, strlen(reverse_sub))); + res = debug_exec(source_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on " + "the source: %s\n"), + reverse_sub, PQerrorMessage(source_conn)); + fully_cleaned = false; + } + else + print_msg(VERBOSITY_NORMAL, + _(" dropped reverse subscription %s on the source\n"), + reverse_sub); + PQclear(res); + pg_free(source_node_name); + } + } + /* Per-peer: drop slot and any reverse subscription */ for (i = 0; i < state->num_peers; i++) { PeerNodeInfo *peer = &state->peers[i]; PGconn *peer_conn; - char reverse_sub[NAMEDATALEN]; + char reverse_sub[NAMEDATALEN * 2 + 8]; if (!peer->dsn || !peer->dsn[0]) continue; @@ -1882,7 +2003,8 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, * touch a same-named resource from an unrelated join, nor report * "incomplete" over a peer that was never touched. */ - if (!peer->slot_created && !peer->reverse_sub_created) + if ((!peer->slot_creation_lsn || !peer->slot_creation_lsn[0]) && + !peer->reverse_sub_created) continue; peer_conn = PQconnectdb(peer->dsn); @@ -1897,7 +2019,8 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, continue; } - if (peer->slot_created && peer->slot_name && peer->slot_name[0]) + if (peer->slot_creation_lsn && peer->slot_creation_lsn[0] && + peer->slot_name && peer->slot_name[0]) { printfPQExpBuffer(query, "SELECT pg_drop_replication_slot(slot_name)" @@ -1959,6 +2082,21 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, destroyPQExpBuffer(query); + /* + * Stop n3's postmaster unconditionally (not gated by --force, which + * only governs removing the data directory). check_data_dir() and + * check_reused_data_dir_is_safe() explicitly support resuming a join + * into this same data_dir after a failed attempt, and that resume + * path (main(), the "start -l ..." pg_ctl call before catchup) assumes + * postgres is not already running here; leaving it up after + * `--cleanup` would make the very next retry fail outright. Every + * drop above already ran while n3 was still reachable -- including + * the source's and each peer's reverse subscription, whose provider + * is n3, so dropping them needs a live connection back to it -- so + * this is just shutdown, not a substitute for them. + */ + stop_postgres_in_data_dir(); + /* * The data directory a partial run may have created via basebackup. * Never touch it without --force. @@ -2485,7 +2623,6 @@ main(int argc, char **argv) drop_slot_if_exists); print_msg(VERBOSITY_DEBUG, _("Source replication slot created: \"%s\"\n"), bidir.source_slot_name); - bidir.source_origin_name = pg_strdup(bidir.source_slot_name); pg_free(source_sub_name); /* @@ -2857,15 +2994,6 @@ main(int argc, char **argv) create_disabled_peer_subscriptions(subscriber_conn, bidir.peers, bidir.num_peers, replication_sets); - /* - * Persist disabled_sub_created for every peer now, not just at - * the top of this block -- a crash during the (possibly long) - * catchup wait below must not leave --cleanup reading a stale - * manifest that still shows every peer's disabled subscription - * as not-yet-created. - */ - write_manifest(&bidir, subscriber_name, db, base_prov_connstr); - print_msg(VERBOSITY_NORMAL, _("Getting catchup target from the source...\n")); target_lsn = get_catchup_target_lsn(prov_connstr); print_msg(VERBOSITY_DEBUG, _("Catchup target LSN: %s\n"), target_lsn); @@ -2876,18 +3004,50 @@ main(int argc, char **argv) "LSN %s\n"), source_sub_name, bidir.source_slot_name, target_lsn); wait_for_catchup(subscriber_conn, source_sub_name, bidir.source_slot_name, target_lsn, bidir.stall_timeout, bidir.max_wait); - pg_free(target_lsn); + + print_msg(VERBOSITY_NORMAL, _("Establishing peer coverage barrier...\n")); + establish_peer_coverage_barrier(&bidir, subscriber_conn, prov_connstr, + remote_info->node_name, source_sub_name, + bidir.source_slot_name, subscriber_name, db, + base_prov_connstr, bidir.stall_timeout, + bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Clearing forwarding on the catchup subscription...\n")); + clear_forwarding(subscriber_conn, prov_connstr, source_sub_name, + bidir.source_slot_name, bidir.stall_timeout, bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Enabling direct peer subscriptions...\n")); + enable_peer_subs(subscriber_conn, bidir.peers, bidir.num_peers, + bidir.stall_timeout, bidir.max_wait); + pg_free(source_sub_name); + + print_msg(VERBOSITY_NORMAL, _("Creating reverse subscriptions...\n")); + create_reverse_subscriptions(&bidir, subscriber_name, sub_connstr, + replication_sets, prov_connstr, + remote_info->node_name, db, base_prov_connstr); + + print_msg(VERBOSITY_NORMAL, _("Waiting for reverse subscriptions to be ready...\n")); + wait_for_reverse_subs_ready(&bidir, subscriber_conn, prov_connstr, + remote_info->node_name, subscriber_name, + bidir.stall_timeout, bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Verifying bidirectional replication...\n")); + verify_bidirectional_dataflow(&bidir, subscriber_conn, prov_connstr, + remote_info->node_name, subscriber_name, + bidir.stall_timeout, bidir.max_wait); + + print_msg(VERBOSITY_NORMAL, _("Lifting read-only mode...\n")); + lift_readonly(subscriber_conn); } PQfinish(subscriber_conn); subscriber_conn = NULL; print_msg(VERBOSITY_NORMAL, - _("Bidirectional join: catchup complete. Node \"%s\" has caught " - "up to the source and forward-tracked every peer's origin; " - "ready for the next phase.\n"), + _("Bidirectional join complete: node \"%s\" is a live, verified " + "bidirectional member of the cluster.\n"), subscriber_name); } else @@ -3337,6 +3497,73 @@ check_reused_data_dir_is_safe(const char *data_dir, RemoteInfo *remoteinfo) pg_free(cf); } +/* + * Create a logical replication slot named slot_name for the spock_output + * plugin and return its creation LSN (caller must free). If a slot of + * that name already exists: drop and recreate it when drop_if_exists is + * set, otherwise die with already_exists_hint appended to the message. + */ +static char * +create_logical_slot(PGconn *conn, const char *slot_name, bool drop_if_exists, + const char *already_exists_hint) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *slot_lsn; + + printfPQExpBuffer(query, + "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check for existing replication slot \"%s\": %s\n"), + slot_name, PQerrorMessage(conn)); + } + + if (PQntuples(res) > 0) + { + PQclear(res); + + if (!drop_if_exists) + die(_("replication slot \"%s\" already exists; %s\n"), + slot_name, already_exists_hint); + + print_msg(VERBOSITY_VERBOSE, + _("dropping existing replication slot \"%s\"...\n"), slot_name); + + printfPQExpBuffer(query, + "SELECT pg_catalog.pg_drop_replication_slot(%s)", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not drop existing replication slot \"%s\": %s\n"), + slot_name, PQerrorMessage(conn)); + } + } + PQclear(res); + + printfPQExpBuffer(query, + "SELECT slot_name, lsn::text" + " FROM pg_create_logical_replication_slot(%s, 'spock_output')", + PQescapeLiteral(conn, slot_name, strlen(slot_name))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not create replication slot \"%s\": %s\n"), + slot_name, PQerrorMessage(conn)); + } + slot_lsn = pg_strdup(PQgetvalue(res, 0, 1)); + PQclear(res); + + destroyPQExpBuffer(query); + return slot_lsn; +} + /* * Initialize replication slots */ @@ -3366,58 +3593,78 @@ initialize_replication_slot(PGconn *conn, char *dbname, slot_name = pstrdup(PQgetvalue(res, 0, 0)); PQclear(res); - resetPQExpBuffer(&query); - - /* Check if the current slot exists. */ - printfPQExpBuffer(&query, - "SELECT 1 FROM pg_catalog.pg_replication_slots WHERE slot_name = %s", - PQescapeLiteral(conn, slot_name, strlen(slot_name))); - - res = debug_exec(conn, query.data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("Could not fetch existing slot information: %s"), PQerrorMessage(conn)); - - /* Drop the existing slot when asked for it or error if it already exists. */ - if (PQntuples(res) > 0) - { - PQclear(res); - resetPQExpBuffer(&query); - - if (!drop_slot_if_exists) - die(_("Slot %s already exists, drop it or use --drop-slot-if-exists to drop it automatically.\n"), - slot_name); - - print_msg(VERBOSITY_VERBOSE, - _("Droping existing slot %s ...\n"), slot_name); + termPQExpBuffer(&query); - printfPQExpBuffer(&query, - "SELECT pg_catalog.pg_drop_replication_slot(%s)", - PQescapeLiteral(conn, slot_name, strlen(slot_name))); + pg_free(create_logical_slot(conn, slot_name, drop_slot_if_exists, + "drop it or use --drop-slot-if-exists to drop " + "it automatically")); - res = debug_exec(conn, query.data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - die(_("Could not drop existing slot %s: %s"), slot_name, - PQerrorMessage(conn)); - } + return slot_name; +} - PQclear(res); - resetPQExpBuffer(&query); +/* + * Create the remote logical replication slot for peer P using its + * pre-generated slot name. Unlike initialize_replication_slot(), a + * pre-existing slot of this name is refused outright rather than + * silently reused or dropped-and-recreated -- at this point in the join + * it can only be a leftover from a botched earlier attempt, and its + * confirmed_flush_lsn cannot be trusted as the WAL-retention floor the + * direct subscription will rely on. Returns the slot's creation LSN as + * text (caller must free). + */ +static char * +create_peer_slot(PGconn *peer_conn, const char *peer_slot_name) +{ + return create_logical_slot(peer_conn, peer_slot_name, false, + "this is unexpected at this stage of the " + "join -- run --cleanup and retry"); +} - /* And finally, create the slot. */ - appendPQExpBuffer(&query, "SELECT pg_create_logical_replication_slot(%s, '%s');", - PQescapeLiteral(conn, slot_name, strlen(slot_name)), - "spock_output"); +/* + * Resolve the origin/slot name for conn's own subscription FROM + * upstream_node_name -- e.g. on the source, the name of its + * subscription to peer P, used to poll the source's own replication + * progress for P (the barrier's Hop 1); or, generalized, a peer's + * subscription to n3 after the reverse subscriptions are created + * (post-cutover dataflow verification). Spock names a subscription's + * replication origin identically to its slot name, so this one value + * serves both the origin lookup and the WHERE external_id = ... poll; + * it is NOT the subscription name, which sub_show_status() needs + * separately, so it is returned too via the nullable sub_name_out + * (caller frees both). Caller must free the return value. + */ +static char * +get_origin_name_for_node(PGconn *conn, const char *upstream_node_name, + const char *conn_label, char **sub_name_out) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *slot_name; - res = debug_exec(conn, query.data); + printfPQExpBuffer(query, + "SELECT s.sub_slot_name, s.sub_name FROM spock.subscription s" + " JOIN spock.node n ON s.sub_origin = n.node_id" + " WHERE n.node_name = %s", + PQescapeLiteral(conn, upstream_node_name, strlen(upstream_node_name))); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - die(_("Could not create replication slot, status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + PQclear(res); + die(_("could not resolve %s's subscription to \"%s\": %s\n"), + conn_label, upstream_node_name, PQerrorMessage(conn)); } - + if (PQntuples(res) != 1) + { + PQclear(res); + die(_("expected exactly one subscription from \"%s\" on %s, found %d\n"), + upstream_node_name, conn_label, PQntuples(res)); + } + slot_name = pg_strdup(PQgetvalue(res, 0, 0)); + if (sub_name_out != NULL) + *sub_name_out = pg_strdup(PQgetvalue(res, 0, 1)); PQclear(res); - termPQExpBuffer(&query); + destroyPQExpBuffer(query); return slot_name; } @@ -3858,14 +4105,43 @@ set_readonly_local(PGconn *conn) } /* - * Restore the replication-set definitions, table memberships, and - * sequence state captured before the catalog strip, now that - * node_create() has given this node an identity again. Without this, - * n3 would accept incoming changes but send nothing back once peers - * create reverse subscriptions later. + * Lift read-only mode. Called only after every subscription (catchup, + * direct peer, and reverse) is verified replicating and bidirectional + * dataflow has actually been proven -- lifting any earlier risks an + * end-user write landing on n3 before it is a fully verified cluster + * member. */ static void -restore_repsets(PGconn *conn, CatalogCapture *capture) +lift_readonly(PGconn *conn) +{ + PGresult *res; + + res = debug_exec(conn, "ALTER SYSTEM SET spock.readonly = 'off'"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + die(_("could not lift spock.readonly: status %s: %s\n"), + PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + } + PQclear(res); + + res = debug_exec(conn, "SELECT pg_reload_conf()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + die(_("could not reload configuration after lifting spock.readonly: %s\n"), + PQerrorMessage(conn)); + } + PQclear(res); +} + +/* + * Restore the replication-set definitions, table memberships, and + * sequence state captured before the catalog strip, now that + * node_create() has given this node an identity again. Without this, + * n3 would accept incoming changes but send nothing back once peers + * create reverse subscriptions later. + */ +static void +restore_repsets(PGconn *conn, CatalogCapture *capture) { PQExpBuffer query = createPQExpBuffer(); PGresult *res; @@ -4405,14 +4681,38 @@ create_disabled_peer_subscriptions(PGconn *subscriber_conn, PeerNodeInfo *peers, "created\n"), peer->slot_name, peer->node_name); } PQclear(res); - - peer->disabled_sub_created = true; } destroyPQExpBuffer(query); destroyPQExpBuffer(repsets); } +/* + * Emit spock.sync_event() on conn and return the resulting LSN as text. + * sync_event() flushes durably before returning, so the LSN is + * guaranteed to arrive at any subscriber via the replication stream -- + * used for the catchup target, both hops of the peer coverage barrier, + * and the post-cutover dataflow verification. Caller must free the + * result. + */ +static char * +get_sync_event_lsn(PGconn *conn, const char *node_label) +{ + PGresult *res; + char *lsn; + + res = debug_exec(conn, "SELECT spock.sync_event()"); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not emit sync_event on %s: %s\n"), node_label, PQerrorMessage(conn)); + } + lsn = pg_strdup(PQgetvalue(res, 0, 0)); + PQclear(res); + + return lsn; +} + /* * A single spock.sync_event() on the source is the catchup target -- it * flushes durably before returning, so the LSN is guaranteed to arrive @@ -4423,45 +4723,83 @@ static char * get_catchup_target_lsn(const char *source_dsn) { PGconn *source_conn; - PGresult *res; char *target_lsn; source_conn = connectdb(source_dsn); - res = debug_exec(source_conn, "SELECT spock.sync_event()"); - if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) - { - PQclear(res); - die(_("could not get catchup target LSN: %s\n"), PQerrorMessage(source_conn)); - } - target_lsn = pg_strdup(PQgetvalue(res, 0, 0)); - PQclear(res); + target_lsn = get_sync_event_lsn(source_conn, "source"); PQfinish(source_conn); return target_lsn; } /* - * Wait for n3's catchup subscription to reach target_lsn. Progress - * watchdog, not a flat wall-clock timeout -- reset the stall clock - * whenever remote_lsn advances at all, since a legitimately large - * catchup can take hours (same shape as wait_primary_connection(), which - * does this for WAL replay). Aborts immediately, without waiting out - * the timeout, if the subscription's own status reports 'disabled' -- - * the signal an unresolvable apply exception leaves behind under - * spock.exception_behaviour = 'sub_disable'; catchup must not be allowed - * to silently stall forever behind a stopped apply worker. + * Shared elapsed-time bookkeeping for the bidirectional-join polling loops + * below (wait_for_origin_progress(), clear_forwarding(), + * wait_for_sub_replicating()) -- each is a progress watchdog, not a flat + * wall-clock timeout, tracking both "time since start" (--max-wait) and + * "time since progress was last observed" (--stall-timeout) separately. + * Each loop still forms and dies with its own message, since what counts + * as "progress" and how to describe it differs per wait; only the time + * math is shared. */ +typedef struct WaitTracker +{ + time_t start_time; + time_t last_progress_time; +} WaitTracker; + static void -wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, - const char *source_slot_name, const char *target_lsn, - int stall_timeout, int max_wait) +wait_tracker_init(WaitTracker *wt) +{ + wt->start_time = time(NULL); + wt->last_progress_time = wt->start_time; +} + +static void +wait_tracker_reset_progress(WaitTracker *wt) +{ + wt->last_progress_time = time(NULL); +} + +static bool +wait_tracker_stalled(const WaitTracker *wt, int stall_timeout) +{ + return stall_timeout > 0 && + (time(NULL) - wt->last_progress_time) >= stall_timeout; +} + +static bool +wait_tracker_timed_out(const WaitTracker *wt, int max_wait) +{ + return max_wait > 0 && (time(NULL) - wt->start_time) >= max_wait; +} + +/* + * Wait for the replication origin named origin_name, on conn, to reach + * target_lsn. Progress watchdog, not a flat wall-clock timeout -- + * reset the stall clock whenever remote_lsn advances at all, since a + * legitimately large wait (a TB-scale catchup, or a laggy peer) can take + * hours (same shape as wait_primary_connection(), which does this for + * WAL replay). If watch_sub_name is non-NULL, also aborts immediately, + * without waiting out the timeout, if that subscription's own status + * reports 'disabled' -- the signal an unresolvable apply exception + * leaves behind under spock.exception_behaviour = 'sub_disable'; a wait + * must not be allowed to silently stall forever behind a stopped apply + * worker. context_label identifies the wait in progress/stall/timeout + * messages (e.g. "catchup", "peer marker (n2 -> n1)"), so a stall is + * attributed to a specific hop rather than reported as generic "waiting". + */ +static void +wait_for_origin_progress(PGconn *conn, const char *origin_name, const char *target_lsn, + const char *watch_sub_name, const char *context_label, + int stall_timeout, int max_wait) { PQExpBuffer query = createPQExpBuffer(); - time_t start_time = time(NULL); - time_t last_progress_time = start_time; + WaitTracker wt; char *last_lsn = NULL; - print_msg(VERBOSITY_VERBOSE, "Waiting for catchup to reach %s...", target_lsn); + wait_tracker_init(&wt); + print_msg(VERBOSITY_VERBOSE, _("Waiting for %s to reach %s..."), context_label, target_lsn); for (;;) { @@ -4471,13 +4809,13 @@ wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, printfPQExpBuffer(query, "SELECT (remote_lsn >= %s::pg_lsn), remote_lsn::text" " FROM pg_replication_origin_status WHERE external_id = %s", - PQescapeLiteral(subscriber_conn, target_lsn, strlen(target_lsn)), - PQescapeLiteral(subscriber_conn, source_slot_name, strlen(source_slot_name))); - res = debug_exec(subscriber_conn, query->data); + PQescapeLiteral(conn, target_lsn, strlen(target_lsn)), + PQescapeLiteral(conn, origin_name, strlen(origin_name))); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); - die(_("could not check catchup progress: %s\n"), PQerrorMessage(subscriber_conn)); + die(_("could not check %s progress: %s\n"), context_label, PQerrorMessage(conn)); } reached = PQntuples(res) == 1 && !PQgetisnull(res, 0, 0) && @@ -4496,7 +4834,7 @@ wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, { pg_free(last_lsn); last_lsn = pg_strdup(cur_lsn); - last_progress_time = time(NULL); + wait_tracker_reset_progress(&wt); } } PQclear(res); @@ -4505,46 +4843,727 @@ wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, * spock.sub_show_status() is the same primitive check_mesh_edges() * relies on for subscription health; 'disabled' here means the * apply worker hit an unresolvable exception and - * spock.exception_behaviour disabled it -- catchup cannot recover + * spock.exception_behaviour disabled it -- this wait cannot recover * from that on its own, so abort now rather than waiting out * stall_timeout/max_wait behind a subscription that will never - * move again. + * move again. Skipped when the caller has no single subscription + * to attribute the wait to (watch_sub_name == NULL). */ - printfPQExpBuffer(query, "SELECT status FROM spock.sub_show_status(%s)", - PQescapeLiteral(subscriber_conn, source_sub_name, strlen(source_sub_name))); - res = debug_exec(subscriber_conn, query->data); + if (watch_sub_name != NULL) + { + printfPQExpBuffer(query, "SELECT status FROM spock.sub_show_status(%s)", + PQescapeLiteral(conn, watch_sub_name, strlen(watch_sub_name))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check subscription \"%s\" status: %s\n"), + watch_sub_name, PQerrorMessage(conn)); + } + if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "disabled") == 0) + { + PQclear(res); + die(_("subscription \"%s\" was disabled during %s, likely by " + "an unresolvable apply exception; this is a hard join " + "failure -- run --cleanup and retry\n"), + watch_sub_name, context_label); + } + PQclear(res); + } + + if (wait_tracker_stalled(&wt, stall_timeout)) + die(_("%s appears stalled: no origin progress for %d second(s) " + "(--stall-timeout)\n"), context_label, stall_timeout); + + if (wait_tracker_timed_out(&wt, max_wait)) + die(_("timed out after %d second(s) waiting for %s to " + "complete (--max-wait)\n"), max_wait, context_label); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + pg_free(last_lsn); + destroyPQExpBuffer(query); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +/* + * Thin wrapper for the catchup wait: the origin polled is sub_n3_n1's, + * named identically to source_slot_name. + */ +static void +wait_for_catchup(PGconn *subscriber_conn, const char *source_sub_name, + const char *source_slot_name, const char *target_lsn, + int stall_timeout, int max_wait) +{ + wait_for_origin_progress(subscriber_conn, source_slot_name, target_lsn, + source_sub_name, "catchup", stall_timeout, max_wait); +} + +/* + * Establish forward-coverage for every peer via a two-hop sync_event() + * barrier, then hand off to Hop 2 once every peer's Hop 1 has landed on + * the source. Direct peer subscriptions stay disabled throughout; only + * forwarding through source_sub_name advances each peer's origin during + * this wait. + * + * Phase A (slot creation) and Hop 1 run interleaved, one peer at a + * time, rather than batching every slot creation first: an unconsumed + * peer slot pins WAL and catalog_xmin on its peer, so minimizing the + * time between a slot's creation and its own Hop 1 landing keeps that + * exposure as short as possible. + * + * Re-verifies replication-set/schema equivalence (P0.5) immediately + * beforehand via the existing check_replication_set_equivalence(), + * since the original precheck ran before the (possibly long) catchup + * wait and topology/schema could have drifted since. + */ +static void +establish_peer_coverage_barrier(BidirectionalState *state, PGconn *n3_conn, + const char *prov_connstr, const char *source_node_name, + const char *source_sub_name, const char *source_slot_name, + const char *subscriber_name, const char *dbname, + const char *base_prov_connstr, + int stall_timeout, int max_wait) +{ + PGconn *source_conn = connectdb(prov_connstr); + char *source_cutover_marker; + int i; + + print_msg(VERBOSITY_NORMAL, + _("Re-verifying replication-set and schema equivalence before " + "the coverage barrier...\n")); + check_replication_set_equivalence(source_conn, source_node_name, + state->peers, state->num_peers); + + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + char *source_slot_for_peer; + char *source_sub_for_peer; + char context_label[NAMEDATALEN * 2 + 32]; + + print_msg(VERBOSITY_NORMAL, + _("Creating replication slot for peer \"%s\"...\n"), peer->node_name); + peer_conn = connectdb(peer->dsn); + + peer->slot_creation_lsn = create_peer_slot(peer_conn, peer->slot_name); + write_manifest(state, subscriber_name, dbname, base_prov_connstr); + print_msg(VERBOSITY_DEBUG, + _("Peer \"%s\" slot \"%s\" created at LSN %s\n"), + peer->node_name, peer->slot_name, peer->slot_creation_lsn); + + peer->peer_marker_lsn = get_sync_event_lsn(peer_conn, peer->node_name); + PQfinish(peer_conn); + + /* + * Hop 1 (P -> source): poll the source's own pre-existing + * subscription to this peer, never n3 -- a peer's own sync_event + * marker is not forwarded through the source (only DML is), so + * polling n3 for it would stall forever on an idle peer. + */ + source_slot_for_peer = get_origin_name_for_node(source_conn, peer->node_name, + "source", &source_sub_for_peer); + print_msg(VERBOSITY_NORMAL, + _("Waiting for the source to catch up to peer \"%s\"...\n"), + peer->node_name); + snprintf(context_label, sizeof(context_label), + "peer marker (%s -> %s)", peer->node_name, source_node_name); + wait_for_origin_progress(source_conn, source_slot_for_peer, peer->peer_marker_lsn, + source_sub_for_peer, context_label, stall_timeout, max_wait); + pg_free(source_slot_for_peer); + pg_free(source_sub_for_peer); + } + + /* + * Hop 2 (source -> n3), once, after every peer's Hop 1 has landed: a + * source-local marker forwards to n3 normally, so reaching it on n3 + * proves n3 has consumed all source WAL ordered before it -- including + * every peer transaction the source had applied by Hop 1. + */ + print_msg(VERBOSITY_NORMAL, + _("Waiting for n3 to reach forward-coverage of all peers...\n")); + source_cutover_marker = get_sync_event_lsn(source_conn, source_node_name); + wait_for_origin_progress(n3_conn, source_slot_name, source_cutover_marker, + source_sub_name, "cutover coverage", stall_timeout, max_wait); + pg_free(source_cutover_marker); + + PQfinish(source_conn); +} + +/* + * Clear forward_origins on the catchup subscription so no peer + * transaction can ever reach n3 by both paths at once -- forwarded via + * the source AND directly via the peer -- once the direct subs are + * enabled. Spock has no origin-based dedup on the apply path, so an + * overlap would double-count delta-apply (conflict-free, additive) + * columns silently; no conflict resolver protects against it. + * + * sub_alter_options() commits the empty forward_origins to the catalog + * immediately and kills the apply worker so it restarts and reconnects + * -- but spock.sub_show_status()'s forward_origins column reads the + * catalog fresh on every call, so it reports the new value the instant + * the ALTER commits, regardless of whether the old, still-forwarding + * worker has actually been replaced yet. That column alone is not + * proof forwarding has stopped. Confirm both: sub_show_status() reports + * 'replicating' with forward_origins = {}, AND the source has accepted + * a *new* walsender connection for this slot (backend_start later than + * one captured before the ALTER) -- since forward_origins is only read + * at connect time (spock_apply.c, spock_connect_replica()), a new + * connection guarantees the empty value is the one actually in effect. + */ +static void +clear_forwarding(PGconn *n3_conn, const char *source_dsn, const char *source_sub_name, + const char *source_slot_name, int stall_timeout, int max_wait) +{ + PGconn *source_conn = connectdb(source_dsn); + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *baseline_backend_start = NULL; + WaitTracker wt; + bool prev_worker_ready = false; + bool prev_reconnected = false; + + wait_tracker_init(&wt); + + printfPQExpBuffer(query, + "SELECT backend_start::text FROM pg_stat_replication" + " WHERE application_name = %s", + PQescapeLiteral(source_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(source_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check existing walsender for \"%s\": %s\n"), + source_slot_name, PQerrorMessage(source_conn)); + } + if (PQntuples(res) == 1 && !PQgetisnull(res, 0, 0)) + baseline_backend_start = pg_strdup(PQgetvalue(res, 0, 0)); + PQclear(res); + + print_msg(VERBOSITY_NORMAL, _("Clearing forwarding on \"%s\"...\n"), source_sub_name); + printfPQExpBuffer(query, + "SELECT spock.sub_alter_options(%s, '{\"forward_origins\": []}'::jsonb)", + PQescapeLiteral(n3_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(n3_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not clear forward_origins on \"%s\": %s\n"), + source_sub_name, PQerrorMessage(n3_conn)); + } + PQclear(res); + + print_msg(VERBOSITY_VERBOSE, + _("Waiting for \"%s\" to reconnect without forwarding..."), source_sub_name); + for (;;) + { + bool worker_ready = false; + bool reconnected = false; + + printfPQExpBuffer(query, + "SELECT status, COALESCE(forward_origins::text, '{}')" + " FROM spock.sub_show_status(%s)", + PQescapeLiteral(n3_conn, source_sub_name, strlen(source_sub_name))); + res = debug_exec(n3_conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { PQclear(res); - die(_("could not check catchup subscription status: %s\n"), - PQerrorMessage(subscriber_conn)); + die(_("could not check subscription \"%s\" status: %s\n"), + source_sub_name, PQerrorMessage(n3_conn)); } if (PQntuples(res) == 1 && strcmp(PQgetvalue(res, 0, 0), "disabled") == 0) { PQclear(res); - die(_("catchup subscription \"%s\" was disabled during catchup, " + die(_("subscription \"%s\" was disabled while clearing forwarding, " "likely by an unresolvable apply exception; this is a hard " "join failure -- run --cleanup and retry\n"), source_sub_name); } + if (PQntuples(res) == 1 && + strcmp(PQgetvalue(res, 0, 0), "replicating") == 0 && + strcmp(PQgetvalue(res, 0, 1), "{}") == 0) + worker_ready = true; PQclear(res); - if (stall_timeout > 0 && (time(NULL) - last_progress_time) >= stall_timeout) - die(_("catchup appears stalled: no origin progress for %d second(s) " - "(--stall-timeout)\n"), stall_timeout); + if (worker_ready) + { + printfPQExpBuffer(query, + "SELECT backend_start::text FROM pg_stat_replication" + " WHERE application_name = %s", + PQescapeLiteral(source_conn, source_slot_name, strlen(source_slot_name))); + res = debug_exec(source_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not check walsender for \"%s\": %s\n"), + source_slot_name, PQerrorMessage(source_conn)); + } + if (PQntuples(res) == 1 && !PQgetisnull(res, 0, 0)) + { + char *cur_backend_start = PQgetvalue(res, 0, 0); - if (max_wait > 0 && (time(NULL) - start_time) >= max_wait) - die(_("timed out after %d second(s) waiting for catchup to " - "complete (--max-wait)\n"), max_wait); + reconnected = (baseline_backend_start == NULL || + strcmp(cur_backend_start, baseline_backend_start) != 0); + } + PQclear(res); + } + + if (worker_ready && reconnected) + break; + + /* + * Reset the stall clock only on an actual transition to true, not + * on every poll where a flag is merely still true -- otherwise a + * worker_ready that becomes (and stays) true while reconnected + * never does would reset the clock forever, defeating + * --stall-timeout entirely. + */ + if ((worker_ready && !prev_worker_ready) || + (reconnected && !prev_reconnected)) + wait_tracker_reset_progress(&wt); + prev_worker_ready = worker_ready; + prev_reconnected = reconnected; + + if (wait_tracker_stalled(&wt, stall_timeout)) + die(_("clearing forwarding on \"%s\" appears stalled: the apply " + "worker has not reconnected without forwarding after %d " + "second(s) (--stall-timeout)\n"), source_sub_name, stall_timeout); + + if (wait_tracker_timed_out(&wt, max_wait)) + die(_("timed out after %d second(s) waiting for forwarding to " + "clear on \"%s\" (--max-wait)\n"), max_wait, source_sub_name); pg_usleep(1000000); /* 1 sec */ print_msg(VERBOSITY_VERBOSE, "."); } - pg_free(last_lsn); + pg_free(baseline_backend_start); destroyPQExpBuffer(query); + PQfinish(source_conn); print_msg(VERBOSITY_VERBOSE, "\n"); } +/* + * Wait for subscription_name on conn to reach status 'replicating'. + * Used after enabling a subscription via the ordinary (non-catchup) + * path, where there is no target LSN to poll against -- just + * confirmation that the apply worker is up. Progress is tracked as + * "the status string changed" rather than an LSN, since there is + * nothing incremental to observe; dies immediately if the subscription + * reports 'disabled' instead. + */ +static void +wait_for_sub_replicating(PGconn *conn, const char *sub_name, int stall_timeout, int max_wait) +{ + PQExpBuffer query = createPQExpBuffer(); + WaitTracker wt; + char *last_status = NULL; + + wait_tracker_init(&wt); + print_msg(VERBOSITY_VERBOSE, _("Waiting for \"%s\" to start replicating..."), sub_name); + + for (;;) + { + PGresult *res; + char *status; + + printfPQExpBuffer(query, "SELECT status FROM spock.sub_show_status(%s)", + PQescapeLiteral(conn, sub_name, strlen(sub_name))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK || PQntuples(res) != 1) + { + PQclear(res); + die(_("could not check subscription \"%s\" status: %s\n"), + sub_name, PQerrorMessage(conn)); + } + status = pg_strdup(PQgetvalue(res, 0, 0)); + PQclear(res); + + if (strcmp(status, "replicating") == 0) + { + pg_free(status); + break; + } + if (strcmp(status, "disabled") == 0) + { + pg_free(status); + die(_("subscription \"%s\" was disabled instead of starting to " + "replicate, likely by an unresolvable apply exception; " + "this is a hard join failure -- run --cleanup and retry\n"), + sub_name); + } + + if (!last_status || strcmp(status, last_status) != 0) + { + pg_free(last_status); + last_status = status; + wait_tracker_reset_progress(&wt); + } + else + pg_free(status); + + if (wait_tracker_stalled(&wt, stall_timeout)) + die(_("\"%s\" appears stalled: status has not changed for %d " + "second(s) (--stall-timeout)\n"), sub_name, stall_timeout); + + if (wait_tracker_timed_out(&wt, max_wait)) + die(_("timed out after %d second(s) waiting for \"%s\" to start " + "replicating (--max-wait)\n"), max_wait, sub_name); + + pg_usleep(1000000); /* 1 sec */ + print_msg(VERBOSITY_VERBOSE, "."); + } + + pg_free(last_status); + destroyPQExpBuffer(query); + print_msg(VERBOSITY_VERBOSE, "\n"); +} + +/* + * Enable each peer's direct subscription, now that forwarding has been + * cleared. Each subscription's origin is already + * durable and at the highest applied peer LSN (advanced in place during + * catchup), so sub_enable() on an already-READY subscription starts the + * apply worker directly with no INIT/sync path; the server clamps to + * the peer slot's floor if the origin is behind it. Enabling only + * after forwarding is cleared is what prevents delivering the same + * peer transaction twice (see clear_forwarding()). + */ +static void +enable_peer_subs(PGconn *n3_conn, PeerNodeInfo *peers, int num_peers, + int stall_timeout, int max_wait) +{ + PQExpBuffer query = createPQExpBuffer(); + int i; + + for (i = 0; i < num_peers; i++) + { + PeerNodeInfo *peer = &peers[i]; + PGresult *res; + + print_msg(VERBOSITY_NORMAL, + _("Enabling direct subscription to peer \"%s\"...\n"), peer->node_name); + printfPQExpBuffer(query, "SELECT spock.sub_enable(%s)", + PQescapeLiteral(n3_conn, peer->sub_name, strlen(peer->sub_name))); + res = debug_exec(n3_conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not enable subscription \"%s\" to peer \"%s\": %s\n"), + peer->sub_name, peer->node_name, PQerrorMessage(n3_conn)); + } + PQclear(res); + + wait_for_sub_replicating(n3_conn, peer->sub_name, stall_timeout, max_wait); + } + + destroyPQExpBuffer(query); +} + +/* + * Create an ordinary enabled subscription named sub_name on conn, + * pointing at provider_dsn, over replication sets repsets (already a + * literal "{...}" array). Used for every reverse subscription -- n3 + * as provider -- created on a peer or on the source. synchronize_data + * := false since the far side already has the data via the physical + * backup and catchup: the INIT/cswp sync path this would otherwise + * take only pauses apply workers on n3, the just-joined node, never on + * an existing live cluster member, so skipping it does not reintroduce + * the pause this design otherwise avoids throughout the forward + * catchup and cutover. + */ +static void +create_subscription_on_conn(PGconn *conn, const char *sub_name, + const char *provider_dsn, const char *repsets, + const char *conn_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + + printfPQExpBuffer(query, + "SELECT spock.sub_create(" + "subscription_name := %s, provider_dsn := %s, " + "replication_sets := %s, " + "synchronize_structure := false, " + "synchronize_data := false)", + PQescapeLiteral(conn, sub_name, strlen(sub_name)), + PQescapeLiteral(conn, provider_dsn, strlen(provider_dsn)), + PQescapeLiteral(conn, repsets, strlen(repsets))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not create subscription \"%s\" on %s: %s\n"), + sub_name, conn_label, PQerrorMessage(conn)); + } + PQclear(res); + + destroyPQExpBuffer(query); +} + +/* + * Create reverse subscriptions -- n3 as provider -- on every peer and + * on the source, so replication becomes bidirectional. + * + * The reverse subscription name -- "sub__" -- + * is not a free choice: cleanup_partial_state() already hard-codes + * this exact pattern when dropping a peer's reverse subscription, so + * getting the direction backwards here would make cleanup silently + * no-op instead of dropping it. disabled_sub_created/reverse_sub_created + * is persisted immediately after each subscription, matching the + * "persist now, not just at the top of this block" discipline already + * used for the catchup and disabled-peer subscriptions: a crash + * partway through this loop must not leave --cleanup reading a + * manifest that still shows an already-created reverse subscription + * as not-yet-created. + */ +static void +create_reverse_subscriptions(BidirectionalState *state, const char *subscriber_name, + const char *n3_dsn, const char *replication_sets, + const char *source_dsn, const char *source_node_name, + const char *dbname, const char *base_prov_connstr) +{ + PQExpBuffer repsets = createPQExpBuffer(); + PQExpBuffer sub_name_buf = createPQExpBuffer(); + int i; + + printfPQExpBuffer(repsets, "{%s}", replication_sets); + + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + char *reverse_sub_name; + + printfPQExpBuffer(sub_name_buf, "sub_%s_%s", peer->node_name, subscriber_name); + reverse_sub_name = pg_strdup(sub_name_buf->data); + + print_msg(VERBOSITY_NORMAL, + _("Creating reverse subscription \"%s\" on peer \"%s\"...\n"), + reverse_sub_name, peer->node_name); + peer_conn = connectdb(peer->dsn); + create_subscription_on_conn(peer_conn, reverse_sub_name, n3_dsn, + repsets->data, peer->node_name); + PQfinish(peer_conn); + + peer->reverse_sub_created = true; + write_manifest(state, subscriber_name, dbname, base_prov_connstr); + pg_free(reverse_sub_name); + } + + { + PGconn *source_conn; + char *reverse_sub_name; + + printfPQExpBuffer(sub_name_buf, "sub_%s_%s", source_node_name, subscriber_name); + reverse_sub_name = pg_strdup(sub_name_buf->data); + + print_msg(VERBOSITY_NORMAL, + _("Creating reverse subscription \"%s\" on the source...\n"), + reverse_sub_name); + source_conn = connectdb(source_dsn); + create_subscription_on_conn(source_conn, reverse_sub_name, n3_dsn, + repsets->data, "the source"); + PQfinish(source_conn); + + state->source_reverse_sub_created = true; + write_manifest(state, subscriber_name, dbname, base_prov_connstr); + pg_free(reverse_sub_name); + } + + destroyPQExpBuffer(repsets); + destroyPQExpBuffer(sub_name_buf); +} + +/* + * Wait for every reverse subscription just created to reach + * 'replicating' -- polled on the node that owns it (a peer or the + * source), not on n3. Also sanity-checks n3's own inbound slot count + * as a belt-and-suspenders cross-check: one inbound slot per peer plus + * the source, matching the reverse subscriptions just created. + */ +static void +wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, int stall_timeout, int max_wait) +{ + PQExpBuffer sub_name_buf = createPQExpBuffer(); + PGconn *source_conn; + PGresult *res; + int i; + int slot_count; + + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn; + char *reverse_sub_name; + + printfPQExpBuffer(sub_name_buf, "sub_%s_%s", peer->node_name, subscriber_name); + reverse_sub_name = pg_strdup(sub_name_buf->data); + + print_msg(VERBOSITY_NORMAL, + _("Waiting for reverse subscription \"%s\" on peer \"%s\"...\n"), + reverse_sub_name, peer->node_name); + peer_conn = connectdb(peer->dsn); + wait_for_sub_replicating(peer_conn, reverse_sub_name, stall_timeout, max_wait); + PQfinish(peer_conn); + pg_free(reverse_sub_name); + } + + { + char *reverse_sub_name; + + printfPQExpBuffer(sub_name_buf, "sub_%s_%s", source_node_name, subscriber_name); + reverse_sub_name = pg_strdup(sub_name_buf->data); + + print_msg(VERBOSITY_NORMAL, + _("Waiting for reverse subscription \"%s\" on the source...\n"), + reverse_sub_name); + source_conn = connectdb(source_dsn); + wait_for_sub_replicating(source_conn, reverse_sub_name, stall_timeout, max_wait); + PQfinish(source_conn); + pg_free(reverse_sub_name); + } + + res = debug_exec(n3_conn, + "SELECT COUNT(*) FROM pg_replication_slots" + " WHERE slot_type = 'logical' AND plugin = 'spock_output'"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + PQclear(res); + die(_("could not count inbound replication slots on n3: %s\n"), + PQerrorMessage(n3_conn)); + } + slot_count = atoi(PQgetvalue(res, 0, 0)); + PQclear(res); + if (slot_count < state->num_peers + 1) + die(_("expected at least %d inbound replication slot(s) on n3 " + "(one per peer plus the source), found %d\n"), + state->num_peers + 1, slot_count); + + destroyPQExpBuffer(sub_name_buf); +} + +/* + * Confirm n3's marker n3_marker has reached remote_conn's own + * subscription from n3 -- i.e. that the reverse subscription actually + * delivers n3's changes outward, not just that sub_create() returned + * success. Used for both a peer and the source below. + */ +static void +verify_dataflow_from_n3(PGconn *remote_conn, const char *remote_node_name, + const char *subscriber_name, const char *n3_marker, + int stall_timeout, int max_wait) +{ + char *remote_origin_for_n3; + char *remote_sub_for_n3; + char context_label[NAMEDATALEN * 2 + 32]; + + remote_origin_for_n3 = get_origin_name_for_node(remote_conn, subscriber_name, + remote_node_name, &remote_sub_for_n3); + snprintf(context_label, sizeof(context_label), + "post-cutover dataflow (n3 -> %s)", remote_node_name); + wait_for_origin_progress(remote_conn, remote_origin_for_n3, n3_marker, + remote_sub_for_n3, context_label, stall_timeout, max_wait); + pg_free(remote_origin_for_n3); + pg_free(remote_sub_for_n3); +} + +/* + * Confirm remote_marker, emitted on a peer or the source, has reached + * n3 via slot_name/sub_name -- i.e. that the direct peer subscription + * or the catchup subscription still delivers inbound changes after + * cutover, not just that it reached 'replicating' once and then + * stalled. + */ +static void +verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, + const char *remote_marker, const char *slot_name, + const char *sub_name, int stall_timeout, int max_wait) +{ + char context_label[NAMEDATALEN * 2 + 32]; + + snprintf(context_label, sizeof(context_label), + "post-cutover dataflow (%s -> n3)", remote_node_name); + wait_for_origin_progress(n3_conn, slot_name, remote_marker, sub_name, + context_label, stall_timeout, max_wait); +} + +/* + * Prove bidirectional dataflow actually works, not just that every + * sub_create()/sub_enable() call so far returned success. Reuses the + * exact two-hop-barrier mechanism -- a fresh sync_event() marker plus + * wait_for_origin_progress() -- in both directions, rather than any + * new SQL surface or a canary DML row the utility would have to pick + * a safe table for: + * + * n3 -> peers/source proves the reverse subscriptions actually deliver + * n3's own changes outward; peers/source -> n3 proves the direct peer + * subscriptions and the catchup subscription still deliver inbound + * changes after cutover, not just that they reached 'replicating' + * once and then stalled. + */ +static void +verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, + const char *source_dsn, const char *source_node_name, + const char *subscriber_name, + int stall_timeout, int max_wait) +{ + char *n3_marker; + int i; + + print_msg(VERBOSITY_NORMAL, + _("Verifying n3's changes reach every peer and the source...\n")); + n3_marker = get_sync_event_lsn(n3_conn, "n3"); + + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn = connectdb(peer->dsn); + + verify_dataflow_from_n3(peer_conn, peer->node_name, subscriber_name, + n3_marker, stall_timeout, max_wait); + PQfinish(peer_conn); + } + + { + PGconn *source_conn = connectdb(source_dsn); + + verify_dataflow_from_n3(source_conn, source_node_name, subscriber_name, + n3_marker, stall_timeout, max_wait); + PQfinish(source_conn); + } + pg_free(n3_marker); + + print_msg(VERBOSITY_NORMAL, + _("Verifying every peer's and the source's changes still reach " + "n3...\n")); + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + PGconn *peer_conn = connectdb(peer->dsn); + char *peer_marker = get_sync_event_lsn(peer_conn, peer->node_name); + + PQfinish(peer_conn); + verify_dataflow_to_n3(n3_conn, peer->node_name, peer_marker, + peer->slot_name, peer->sub_name, stall_timeout, max_wait); + pg_free(peer_marker); + } + + { + PGconn *source_conn = connectdb(source_dsn); + char *source_marker = get_sync_event_lsn(source_conn, source_node_name); + + PQfinish(source_conn); + verify_dataflow_to_n3(n3_conn, source_node_name, source_marker, + state->source_slot_name, NULL, stall_timeout, max_wait); + pg_free(source_marker); + } +} + /* * Initialize new remote identifier to specific position. */ From cb8923ac66fd1ff82bf37f22c08cb8ea21504eb6 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Mon, 24 Aug 2026 16:45:37 +0500 Subject: [PATCH 12/14] spock_create_subscriber: unify peer/source loops in the reverse-sub phase create_reverse_subscriptions(), wait_for_reverse_subs_ready(), and verify_bidirectional_dataflow() each looped over every peer and then repeated a near-identical block for the source. By this point in the join the source is handled exactly like a peer in all three operations, so the split was pure duplication. Add build_reverse_sub_targets(): a small local list of {node_name, dsn, slot_name, sub_name, reverse_sub_created} built from the peers array plus the source's own equivalents, letting all three functions loop once instead of peer-loop-then-source-block. Scoped to just these three functions. reverse_sub_created is a *value* field on PeerNodeInfo (and a separate scalar on BidirectionalState for the source), so the target list cannot copy entries by value -- writing through a copy would never reach write_manifest(), silently breaking --cleanup after a crash. It instead aliases the real storage via a bool * for that one field. Also factor the duplicated "sub__" name construction into reverse_sub_name_for(), and point cleanup_partial_state()'s two source/peer reverse-subscription-name buffers at it too -- they predate this commit and were sized NAMEDATALEN, one byte short of what "sub__" can need for max-length identifiers, silently truncating and making spock.sub_drop(..., ifexists := true) quietly no-op on a name that was never the real one. --- .../spock_create_subscriber.c | 777 +++++++++--------- 1 file changed, 410 insertions(+), 367 deletions(-) diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 9fa3c1b5d..8c4ac2fa6 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -218,6 +218,7 @@ static char *initialize_replication_slot(PGconn *conn, char *dbname, char *provider_node_name, char *subscription_name, bool drop_slot_if_exists); static char *create_peer_slot(PGconn *peer_conn, const char *peer_slot_name); +static char *sub_name_for(const char *local_node_name, const char *provider_node_name); static char *get_origin_name_for_node(PGconn *conn, const char *upstream_node_name, const char *conn_label, char **sub_name_out); static void spock_subscribe(PGconn *conn, char *subscriber_name, @@ -269,7 +270,6 @@ static void check_mesh_edges(PGconn *conn, const char *this_node_name, char **all_names, int total_nodes); static void check_peer_identity(PGconn *peer_conn, const char *expected_name); static void check_replication_set_equivalence(PGconn *source_conn, - const char *source_node_name, PeerNodeInfo *peers, int num_peers); static void write_manifest(BidirectionalState *state, const char *subscriber_name, const char *dbname, const char *source_dsn); @@ -336,9 +336,9 @@ static void verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, const char *sub_name, int stall_timeout, int max_wait); static void verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, const char *source_dsn, const char *source_node_name, - const char *subscriber_name, int stall_timeout, int max_wait); -static void set_readonly_local(PGconn *conn); -static void lift_readonly(PGconn *conn); + const char *source_sub_name, const char *subscriber_name, + int stall_timeout, int max_wait); +static void set_spock_readonly(PGconn *conn, const char *value); static Oid get_local_node_id(PGconn *conn); static PGconn * @@ -439,15 +439,9 @@ discover_peer_nodes(PGconn *source_conn, const char *source_node_name, for (i = 0; i < npeers; i++) { - PQExpBuffer sub_name_buf = createPQExpBuffer(); - peers[i].node_name = pg_strdup(PQgetvalue(res, i, 0)); peers[i].dsn = pg_strdup(PQgetvalue(res, i, 1)); - - appendPQExpBuffer(sub_name_buf, "sub_%s_%s", - subscriber_name, peers[i].node_name); - peers[i].sub_name = pg_strdup(sub_name_buf->data); - destroyPQExpBuffer(sub_name_buf); + peers[i].sub_name = sub_name_for(subscriber_name, peers[i].node_name); paramValues[0] = dbname; paramValues[1] = peers[i].node_name; @@ -932,8 +926,7 @@ build_selected_set_name_filter(PGconn *conn) * each peer; reject any mismatch or missing/extra set on either side. */ static void -check_replication_set_equivalence(PGconn *source_conn, const char *source_node_name, - PeerNodeInfo *peers, int num_peers) +check_replication_set_equivalence(PGconn *source_conn, PeerNodeInfo *peers, int num_peers) { Oid source_nodeid = get_local_node_id(source_conn); char *selected_filter = build_selected_set_name_filter(source_conn); @@ -1015,7 +1008,6 @@ check_replication_set_equivalence(PGconn *source_conn, const char *source_node_n free_repset_fingerprints(source_fps, num_source_fps); pg_free(selected_filter); - (void) source_node_name; } /* @@ -1120,7 +1112,7 @@ check_preconditions(PGconn *source_conn, const char *source_node_name, } /* Replication-set & schema equivalence: run once the mesh is sound. */ - check_replication_set_equivalence(source_conn, source_node_name, peers, num_peers); + check_replication_set_equivalence(source_conn, peers, num_peers); for (i = 0; i < total_nodes; i++) pg_free(all_names[i]); @@ -1793,109 +1785,215 @@ check_sysid_matches(PGconn *conn, const char *expected_sysid) } /* - * Idempotently remove bidirectional join state from all reachable nodes. - * Connects to the subscriber (n3) itself, the source, and each peer; - * drops n3's own catchup/disabled-peer subscriptions, replication slots, - * and reverse subscriptions created during a previous join attempt. - * spock.sub_drop() on n3 kills that subscription's local apply worker and - * drops the matching remote slot on its origin itself, so n3 never needs - * to be stopped just to release a slot it holds open elsewhere. - * Connectivity and drop failures are logged as warnings, not fatal, so - * cleanup attempts every remaining resource -- but each failure is - * tracked, and the function returns true only if every recorded resource - * was confirmed gone. The manifest/sidecar record (the only way to - * retry) is removed only on a true return; an incomplete cleanup keeps - * it and the caller exits non-zero. + * Outcome of an idempotent drop-if-exists cleanup operation. A plain bool + * can't tell "nothing to do" apart from "confirmed gone", which callers + * that gate further action (or --cleanup's own retry decision) on actual + * removal need to distinguish from a query failure. + */ +typedef enum +{ + CLEANUP_DROP_FAILED, /* query failed; removal not confirmed */ + CLEANUP_DROP_ABSENT, /* query succeeded; resource did not exist */ + CLEANUP_DROP_REMOVED /* query succeeded; resource was dropped */ +} CleanupDropOutcome; + +/* + * Drop the replication slot named slot_name on conn if it exists. + * node_label identifies the node in progress/warning messages. + */ +static CleanupDropOutcome +drop_logical_slot_if_exists(PGconn *conn, const char *slot_name, const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, + "SELECT pg_drop_replication_slot(slot_name)" + " FROM pg_replication_slots" + " WHERE slot_name = '%s'", + slot_name); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop slot %s on %s: %s\n"), + slot_name, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (PQntuples(res) > 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped slot %s on %s\n"), slot_name, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop the reverse subscription named "sub__" + * on conn, via spock.sub_drop(..., ifexists := true) -- an absent + * subscription is not an error. node_label identifies the node in + * progress/warning messages. spock.sub_drop() itself returns whether it + * found and dropped a subscription, which is read back here to distinguish + * CLEANUP_DROP_ABSENT from CLEANUP_DROP_REMOVED; CLEANUP_DROP_FAILED (and a + * warning) is returned only on an actual query failure. spock.sub_drop() + * is declared RETURNS oid in the catalog (sql/spock--6.0.0.sql) even + * though its C implementation (spock_drop_subscription) returns a bool + * Datum, so the wire text is "0"/"1", not "f"/"t" -- compare against "0" + * rather than checking for boolean-formatted text. + */ +static CleanupDropOutcome +drop_reverse_sub(PGconn *conn, const char *node_name, const char *subscriber_name, + const char *node_label) +{ + PQExpBuffer query = createPQExpBuffer(); + PGresult *res; + char *reverse_sub = sub_name_for(node_name, subscriber_name); + CleanupDropOutcome outcome; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(conn, reverse_sub, strlen(reverse_sub))); + res = debug_exec(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not drop reverse subscription %s on %s: %s\n"), + reverse_sub, node_label, PQerrorMessage(conn)); + outcome = CLEANUP_DROP_FAILED; + } + else if (strcmp(PQgetvalue(res, 0, 0), "0") != 0) + { + print_msg(VERBOSITY_NORMAL, + _(" dropped reverse subscription %s on %s\n"), + reverse_sub, node_label); + outcome = CLEANUP_DROP_REMOVED; + } + else + outcome = CLEANUP_DROP_ABSENT; + PQclear(res); + + pg_free(reverse_sub); + destroyPQExpBuffer(query); + return outcome; +} + +/* + * Drop every subscription this run may have created on n3 itself: the + * catchup subscription to the source and any disabled peer subscriptions. + * A freshly-provisioned n3 has no other legitimate spock.subscription + * rows, so it's safe to drop everything found -- but only once node_sysid + * confirms node_dsn still reaches that same n3, since a manifest can + * outlive the node it describes (DNS change, load balancer, reused port). + * node_dsn is only set once node_create() has run, so its absence just + * means there's nothing on n3 yet, and this returns true without doing + * anything. spock.sub_drop() on n3 kills that subscription's local apply + * worker and drops the matching remote slot on its origin itself, so n3 + * never needs to be stopped just to release a slot it holds open + * elsewhere. */ static bool -cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, - const char *dbname, const char *source_dsn, - bool force_rm_datadir) +cleanup_verified_subscriber_node(BidirectionalState *state, const char *subscriber_name) { - PGconn *source_conn; PGconn *n3_conn; PGresult *res; - PQExpBuffer query = createPQExpBuffer(); + PQExpBuffer query; int i; bool fully_cleaned = true; - print_msg(VERBOSITY_NORMAL, - _("Cleaning up partial bidirectional join state ...\n")); + if (!state->node_dsn || !state->node_dsn[0]) + return true; - /* - * Drop any subscriptions this run created on n3 itself: the catchup - * subscription to the source and any disabled peer subscriptions. A - * freshly-provisioned n3 has no other legitimate spock.subscription - * rows, so it's safe to drop everything found -- but only once - * node_sysid confirms node_dsn still reaches that same n3, since a - * manifest can outlive the node it describes (DNS change, load - * balancer, reused port). node_dsn is only set once node_create() - * has run, so its absence just means there's nothing on n3 yet. - */ - if (state->node_dsn && state->node_dsn[0]) + n3_conn = PQconnectdb(state->node_dsn); + if (PQstatus(n3_conn) != CONNECTION_OK) { - n3_conn = PQconnectdb(state->node_dsn); - if (PQstatus(n3_conn) != CONNECTION_OK) - { - print_msg(VERBOSITY_NORMAL, - _("warning: cannot connect to subscriber \"%s\"; its " - "subscription(s) may still exist: %s\n"), - subscriber_name, PQerrorMessage(n3_conn)); - fully_cleaned = false; - PQfinish(n3_conn); - } - else if (!state->node_sysid || !state->node_sysid[0] || - !check_sysid_matches(n3_conn, state->node_sysid)) - { - print_msg(VERBOSITY_NORMAL, - _("warning: node_dsn for subscriber \"%s\" cannot be " - "confirmed to still identify the node this run " - "created (missing or mismatched system identifier); " - "refusing to drop subscriptions there. Investigate " - "manually.\n"), subscriber_name); - fully_cleaned = false; - PQfinish(n3_conn); - } - else + print_msg(VERBOSITY_NORMAL, + _("warning: cannot connect to subscriber \"%s\"; its " + "subscription(s) may still exist: %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + PQfinish(n3_conn); + return false; + } + + if (!state->node_sysid || !state->node_sysid[0] || + !check_sysid_matches(n3_conn, state->node_sysid)) + { + print_msg(VERBOSITY_NORMAL, + _("warning: node_dsn for subscriber \"%s\" cannot be " + "confirmed to still identify the node this run " + "created (missing or mismatched system identifier); " + "refusing to drop subscriptions there. Investigate " + "manually.\n"), subscriber_name); + PQfinish(n3_conn); + return false; + } + + query = createPQExpBuffer(); + res = debug_exec(n3_conn, "SELECT sub_name FROM spock.subscription"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) + { + for (i = 0; i < PQntuples(res); i++) { - res = debug_exec(n3_conn, "SELECT sub_name FROM spock.subscription"); - if (PQresultStatus(res) == PGRES_TUPLES_OK) - { - for (i = 0; i < PQntuples(res); i++) - { - char *sub_name = PQgetvalue(res, i, 0); - PGresult *drop_res; - - printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", - PQescapeLiteral(n3_conn, sub_name, strlen(sub_name))); - drop_res = debug_exec(n3_conn, query->data); - if (PQresultStatus(drop_res) == PGRES_TUPLES_OK) - print_msg(VERBOSITY_NORMAL, - _(" dropped subscriber subscription %s\n"), - sub_name); - else - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not drop subscriber " - "subscription %s: %s\n"), - sub_name, PQerrorMessage(n3_conn)); - fully_cleaned = false; - } - PQclear(drop_res); - } - } + char *sub_name = PQgetvalue(res, i, 0); + PGresult *drop_res; + + printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", + PQescapeLiteral(n3_conn, sub_name, strlen(sub_name))); + drop_res = debug_exec(n3_conn, query->data); + if (PQresultStatus(drop_res) == PGRES_TUPLES_OK) + print_msg(VERBOSITY_NORMAL, + _(" dropped subscriber subscription %s\n"), + sub_name); else { print_msg(VERBOSITY_NORMAL, - _("warning: could not list subscriptions on " - "subscriber \"%s\": %s\n"), - subscriber_name, PQerrorMessage(n3_conn)); + _("warning: could not drop subscriber " + "subscription %s: %s\n"), + sub_name, PQerrorMessage(n3_conn)); fully_cleaned = false; } - PQclear(res); - PQfinish(n3_conn); + PQclear(drop_res); } } + else + { + print_msg(VERBOSITY_NORMAL, + _("warning: could not list subscriptions on " + "subscriber \"%s\": %s\n"), + subscriber_name, PQerrorMessage(n3_conn)); + fully_cleaned = false; + } + PQclear(res); + destroyPQExpBuffer(query); + PQfinish(n3_conn); + + return fully_cleaned; +} + +/* + * Drop resources this run may have created on the source and each peer: + * the source's replication slot and reverse subscription (if recorded as + * created), and each peer's replication slot and reverse subscription. + * Unlike n3 (a freshly-provisioned node with no unrelated subscriptions, + * see cleanup_verified_subscriber_node()), the source and peers have their + * own pre-existing state that must not be touched -- so every drop here is + * gated by a flag or an LSN this run itself recorded, and reverse + * subscriptions are targeted by name via drop_reverse_sub() rather than + * dropping everything found. + */ +static bool +cleanup_upstream_node_resources(BidirectionalState *state, const char *subscriber_name, + const char *source_dsn) +{ + PGconn *source_conn; + PGresult *res; + int i; + bool fully_cleaned = true; source_conn = PQconnectdb(source_dsn); if (PQstatus(source_conn) != CONNECTION_OK) @@ -1915,36 +2013,18 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, /* Drop source replication slot if it was created */ if (source_conn && state->source_slot_name && state->source_slot_name[0]) { - printfPQExpBuffer(query, - "SELECT pg_drop_replication_slot(slot_name)" - " FROM pg_replication_slots" - " WHERE slot_name = '%s'", - state->source_slot_name); - res = debug_exec(source_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK) - { - if (PQntuples(res) > 0) - print_msg(VERBOSITY_NORMAL, - _(" dropped source slot %s\n"), - state->source_slot_name); - } - else - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not drop source slot %s: %s\n"), - state->source_slot_name, PQerrorMessage(source_conn)); + if (drop_logical_slot_if_exists(source_conn, state->source_slot_name, + "the source") == CLEANUP_DROP_FAILED) fully_cleaned = false; - } - PQclear(res); } /* * Drop the reverse subscription on the source if this run recorded - * having created it. Unlike n3 (whose n3-side block above - * drops every subscription it finds, since a fresh n3 has no other - * legitimate ones), the source has its own pre-existing, unrelated - * subscriptions that must not be touched -- so this is gated by the - * flag and targets the specific reverse subscription by name. + * having created it. Unlike n3 (whose block drops every subscription + * it finds, since a fresh n3 has no other legitimate ones), the source + * has its own pre-existing, unrelated subscriptions that must not be + * touched -- so this is gated by the flag and targets the specific + * reverse subscription by name. */ if (source_conn && state->source_reverse_sub_created) { @@ -1961,27 +2041,11 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, else { char *source_node_name = pg_strdup(PQgetvalue(res, 0, 0)); - char reverse_sub[NAMEDATALEN * 2 + 8]; PQclear(res); - snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", - source_node_name, subscriber_name); - printfPQExpBuffer(query, "SELECT spock.sub_drop(%s, true)", - PQescapeLiteral(source_conn, reverse_sub, strlen(reverse_sub))); - res = debug_exec(source_conn, query->data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not drop reverse subscription %s on " - "the source: %s\n"), - reverse_sub, PQerrorMessage(source_conn)); + if (drop_reverse_sub(source_conn, source_node_name, subscriber_name, + "the source") == CLEANUP_DROP_FAILED) fully_cleaned = false; - } - else - print_msg(VERBOSITY_NORMAL, - _(" dropped reverse subscription %s on the source\n"), - reverse_sub); - PQclear(res); pg_free(source_node_name); } } @@ -1991,7 +2055,6 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, { PeerNodeInfo *peer = &state->peers[i]; PGconn *peer_conn; - char reverse_sub[NAMEDATALEN * 2 + 8]; if (!peer->dsn || !peer->dsn[0]) continue; @@ -2022,28 +2085,9 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, if (peer->slot_creation_lsn && peer->slot_creation_lsn[0] && peer->slot_name && peer->slot_name[0]) { - printfPQExpBuffer(query, - "SELECT pg_drop_replication_slot(slot_name)" - " FROM pg_replication_slots" - " WHERE slot_name = '%s'", - peer->slot_name); - res = debug_exec(peer_conn, query->data); - if (PQresultStatus(res) == PGRES_TUPLES_OK) - { - if (PQntuples(res) > 0) - print_msg(VERBOSITY_NORMAL, - _(" dropped peer slot %s on %s\n"), - peer->slot_name, peer->node_name); - } - else - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not drop peer slot %s on %s: %s\n"), - peer->slot_name, peer->node_name, - PQerrorMessage(peer_conn)); + if (drop_logical_slot_if_exists(peer_conn, peer->slot_name, + peer->node_name) == CLEANUP_DROP_FAILED) fully_cleaned = false; - } - PQclear(res); } /* @@ -2055,21 +2099,9 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, */ if (peer->reverse_sub_created) { - snprintf(reverse_sub, sizeof(reverse_sub), "sub_%s_%s", - peer->node_name, subscriber_name); - printfPQExpBuffer(query, - "SELECT spock.sub_drop(%s, true)", - PQescapeLiteral(peer_conn, reverse_sub, strlen(reverse_sub))); - res = debug_exec(peer_conn, query->data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - print_msg(VERBOSITY_NORMAL, - _("warning: could not drop reverse subscription %s on " - "%s: %s\n"), - reverse_sub, peer->node_name, PQerrorMessage(peer_conn)); + if (drop_reverse_sub(peer_conn, peer->node_name, subscriber_name, + peer->node_name) == CLEANUP_DROP_FAILED) fully_cleaned = false; - } - PQclear(res); } PQfinish(peer_conn); @@ -2080,7 +2112,35 @@ cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, if (source_conn) PQfinish(source_conn); - destroyPQExpBuffer(query); + return fully_cleaned; +} + +/* + * Idempotently remove bidirectional join state from all reachable nodes: + * n3 itself, the source, and each peer, via + * cleanup_verified_subscriber_node() and cleanup_upstream_node_resources(). + * Connectivity and drop failures are logged as warnings, not fatal, so + * cleanup attempts every remaining resource -- but each failure is + * tracked, and the function returns true only if every recorded resource + * was confirmed gone. The manifest/sidecar record (the only way to + * retry) is removed only on a true return; an incomplete cleanup keeps + * it and the caller exits non-zero. + */ +static bool +cleanup_partial_state(BidirectionalState *state, const char *subscriber_name, + const char *dbname, const char *source_dsn, + bool force_rm_datadir) +{ + bool fully_cleaned = true; + + print_msg(VERBOSITY_NORMAL, + _("Cleaning up partial bidirectional join state ...\n")); + + if (!cleanup_verified_subscriber_node(state, subscriber_name)) + fully_cleaned = false; + + if (!cleanup_upstream_node_resources(state, subscriber_name, source_dsn)) + fully_cleaned = false; /* * Stop n3's postmaster unconditionally (not gated by --force, which @@ -2571,7 +2631,6 @@ main(int argc, char **argv) */ if (bidir.enabled) { - PQExpBuffer sub_name_buf = createPQExpBuffer(); char *source_sub_name; /* @@ -2606,10 +2665,7 @@ main(int argc, char **argv) if (use_existing_data_dir) check_reused_data_dir_is_safe(data_dir, remote_info); - appendPQExpBuffer(sub_name_buf, "sub_%s_%s", - subscriber_name, remote_info->node_name); - source_sub_name = pg_strdup(sub_name_buf->data); - destroyPQExpBuffer(sub_name_buf); + source_sub_name = sub_name_for(subscriber_name, remote_info->node_name); print_msg(VERBOSITY_NORMAL, _("Creating source replication slot in database %s ...\n"), db); @@ -2953,7 +3009,7 @@ main(int argc, char **argv) } print_msg(VERBOSITY_NORMAL, _("Setting spock.readonly = 'local'...\n")); - set_readonly_local(subscriber_conn); + set_spock_readonly(subscriber_conn, "local"); /* Restore what was captured before the catalog strip. */ print_msg(VERBOSITY_NORMAL, _("Restoring replication set state...\n")); @@ -2969,14 +3025,10 @@ main(int argc, char **argv) write_manifest(&bidir, subscriber_name, db, base_prov_connstr); { - PQExpBuffer sub_name_buf = createPQExpBuffer(); char *source_sub_name; char *target_lsn; - appendPQExpBuffer(sub_name_buf, "sub_%s_%s", - subscriber_name, remote_info->node_name); - source_sub_name = pg_strdup(sub_name_buf->data); - destroyPQExpBuffer(sub_name_buf); + source_sub_name = sub_name_for(subscriber_name, remote_info->node_name); print_msg(VERBOSITY_NORMAL, _("Creating catchup subscription to the source...\n")); print_msg(VERBOSITY_DEBUG, @@ -3021,8 +3073,6 @@ main(int argc, char **argv) enable_peer_subs(subscriber_conn, bidir.peers, bidir.num_peers, bidir.stall_timeout, bidir.max_wait); - pg_free(source_sub_name); - print_msg(VERBOSITY_NORMAL, _("Creating reverse subscriptions...\n")); create_reverse_subscriptions(&bidir, subscriber_name, sub_connstr, replication_sets, prov_connstr, @@ -3035,11 +3085,13 @@ main(int argc, char **argv) print_msg(VERBOSITY_NORMAL, _("Verifying bidirectional replication...\n")); verify_bidirectional_dataflow(&bidir, subscriber_conn, prov_connstr, - remote_info->node_name, subscriber_name, - bidir.stall_timeout, bidir.max_wait); + remote_info->node_name, source_sub_name, + subscriber_name, bidir.stall_timeout, bidir.max_wait); + + pg_free(source_sub_name); print_msg(VERBOSITY_NORMAL, _("Lifting read-only mode...\n")); - lift_readonly(subscriber_conn); + set_spock_readonly(subscriber_conn, "off"); } PQfinish(subscriber_conn); @@ -4078,56 +4130,37 @@ remove_unwanted_data_bidir(PGconn *conn, CatalogCapture *capture) } /* - * Immediately after node_create, make the new node read-only to - * non-superuser clients -- there must be no window where n3 is - * reachable/writable before this lands. - */ -static void -set_readonly_local(PGconn *conn) -{ - PGresult *res; - - res = debug_exec(conn, "ALTER SYSTEM SET spock.readonly = 'local'"); - if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - die(_("could not set spock.readonly: status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); - } - PQclear(res); - - res = debug_exec(conn, "SELECT pg_reload_conf()"); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - die(_("could not reload configuration after setting spock.readonly: %s\n"), - PQerrorMessage(conn)); - } - PQclear(res); -} - -/* - * Lift read-only mode. Called only after every subscription (catchup, - * direct peer, and reverse) is verified replicating and bidirectional - * dataflow has actually been proven -- lifting any earlier risks an + * Set spock.readonly to value ("local" or "off") and reload the + * configuration so it takes effect immediately. Used both immediately + * after node_create(), to lock n3 down to non-superuser clients (value = + * "local" -- there must be no window where n3 is reachable/writable + * before this lands), and, at the very end of the join, to lift it + * (value = "off", called only after every subscription -- catchup, + * direct peer, and reverse -- is verified replicating and bidirectional + * dataflow has actually been proven; lifting any earlier risks an * end-user write landing on n3 before it is a fully verified cluster - * member. + * member). */ static void -lift_readonly(PGconn *conn) +set_spock_readonly(PGconn *conn, const char *value) { + PQExpBuffer query = createPQExpBuffer(); PGresult *res; - res = debug_exec(conn, "ALTER SYSTEM SET spock.readonly = 'off'"); + printfPQExpBuffer(query, "ALTER SYSTEM SET spock.readonly = '%s'", value); + res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_COMMAND_OK) { - die(_("could not lift spock.readonly: status %s: %s\n"), - PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); + die(_("could not set spock.readonly to '%s': status %s: %s\n"), + value, PQresStatus(PQresultStatus(res)), PQresultErrorMessage(res)); } PQclear(res); + destroyPQExpBuffer(query); res = debug_exec(conn, "SELECT pg_reload_conf()"); if (PQresultStatus(res) != PGRES_TUPLES_OK) { - die(_("could not reload configuration after lifting spock.readonly: %s\n"), + die(_("could not reload configuration after setting spock.readonly: %s\n"), PQerrorMessage(conn)); } PQclear(res); @@ -4154,32 +4187,19 @@ restore_repsets(PGconn *conn, CatalogCapture *capture) strcmp(s->set_name, "default_insert_only") == 0 || strcmp(s->set_name, "ddl_sql") == 0); - if (builtin) - printfPQExpBuffer(query, - "SELECT spock.repset_alter(" - "set_name := %s, " - "replicate_insert := %s, " - "replicate_update := %s, " - "replicate_delete := %s, " - "replicate_truncate := %s)", - PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), - s->replicate_insert ? "true" : "false", - s->replicate_update ? "true" : "false", - s->replicate_delete ? "true" : "false", - s->replicate_truncate ? "true" : "false"); - else - printfPQExpBuffer(query, - "SELECT spock.repset_create(" - "set_name := %s, " - "replicate_insert := %s, " - "replicate_update := %s, " - "replicate_delete := %s, " - "replicate_truncate := %s)", - PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), - s->replicate_insert ? "true" : "false", - s->replicate_update ? "true" : "false", - s->replicate_delete ? "true" : "false", - s->replicate_truncate ? "true" : "false"); + printfPQExpBuffer(query, + "SELECT spock.%s(" + "set_name := %s, " + "replicate_insert := %s, " + "replicate_update := %s, " + "replicate_delete := %s, " + "replicate_truncate := %s)", + builtin ? "repset_alter" : "repset_create", + PQescapeLiteral(conn, s->set_name, strlen(s->set_name)), + s->replicate_insert ? "true" : "false", + s->replicate_update ? "true" : "false", + s->replicate_delete ? "true" : "false", + s->replicate_truncate ? "true" : "false"); res = debug_exec(conn, query->data); if (PQresultStatus(res) != PGRES_TUPLES_OK) { @@ -4934,8 +4954,7 @@ establish_peer_coverage_barrier(BidirectionalState *state, PGconn *n3_conn, print_msg(VERBOSITY_NORMAL, _("Re-verifying replication-set and schema equivalence before " "the coverage barrier...\n")); - check_replication_set_equivalence(source_conn, source_node_name, - state->peers, state->num_peers); + check_replication_set_equivalence(source_conn, state->peers, state->num_peers); for (i = 0; i < state->num_peers; i++) { @@ -5303,15 +5322,90 @@ create_subscription_on_conn(PGconn *conn, const char *sub_name, destroyPQExpBuffer(query); } +/* + * A node -- peer or source -- in the reverse-subscription phase after + * cutover: create a reverse subscription, wait for it, verify + * dataflow in both directions. The source is handled identically to + * a peer in all three, so one list lets each function loop once + * instead of a peer loop plus a near-identical source-only block. + * Fields borrow from existing storage (a PeerNodeInfo entry, or the + * source_dsn/source_node_name/state arguments); only the array itself + * is owned by the caller. + */ +typedef struct ReverseSubTarget +{ + const char *node_name; + const char *dsn; /* connection string for this node */ + const char *slot_name; /* this node's inbound slot on n3 */ + const char *sub_name; /* n3's subscription from this node, or NULL */ + bool *reverse_sub_created; /* where to persist reverse_sub_created */ +} ReverseSubTarget; + +/* + * Build the list above: every peer, in discovery order, then the + * source. *num_targets_out is set to state->num_peers + 1. + * source_sub_name is the source's own subscription name -- the + * catchup subscription, still carrying source -> n3 dataflow after + * cutover -- or NULL if unused (only verify_bidirectional_dataflow()'s + * inbound leg watches it for 'disabled'). Caller must pg_free() the + * returned array. + */ +static ReverseSubTarget * +build_reverse_sub_targets(BidirectionalState *state, const char *source_dsn, + const char *source_node_name, const char *source_sub_name, + int *num_targets_out) +{ + int num_targets = state->num_peers + 1; + ReverseSubTarget *targets = pg_malloc(num_targets * sizeof(ReverseSubTarget)); + int i; + + for (i = 0; i < state->num_peers; i++) + { + PeerNodeInfo *peer = &state->peers[i]; + + targets[i].node_name = peer->node_name; + targets[i].dsn = peer->dsn; + targets[i].slot_name = peer->slot_name; + targets[i].sub_name = peer->sub_name; + targets[i].reverse_sub_created = &peer->reverse_sub_created; + } + + targets[state->num_peers].node_name = source_node_name; + targets[state->num_peers].dsn = source_dsn; + targets[state->num_peers].slot_name = state->source_slot_name; + targets[state->num_peers].sub_name = source_sub_name; + targets[state->num_peers].reverse_sub_created = &state->source_reverse_sub_created; + + *num_targets_out = num_targets; + return targets; +} + +/* + * "sub__": Spock's subscription + * naming convention, used both for a peer's or the source's reverse + * subscription to n3 (local = that node, provider = n3) and for n3's + * own subscription to a peer or the source (local = n3, provider = + * that node) -- not a free choice: cleanup_partial_state() hard-codes + * this exact pattern when dropping a reverse subscription, so getting + * the direction backwards would make cleanup silently no-op instead + * of dropping it. Caller must pg_free() the result. + */ +static char * +sub_name_for(const char *local_node_name, const char *provider_node_name) +{ + PQExpBuffer buf = createPQExpBuffer(); + char *name; + + printfPQExpBuffer(buf, "sub_%s_%s", local_node_name, provider_node_name); + name = pg_strdup(buf->data); + destroyPQExpBuffer(buf); + + return name; +} + /* * Create reverse subscriptions -- n3 as provider -- on every peer and - * on the source, so replication becomes bidirectional. - * - * The reverse subscription name -- "sub__" -- - * is not a free choice: cleanup_partial_state() already hard-codes - * this exact pattern when dropping a peer's reverse subscription, so - * getting the direction backwards here would make cleanup silently - * no-op instead of dropping it. disabled_sub_created/reverse_sub_created + * on the source, so replication becomes bidirectional. reverse_sub_created * is persisted immediately after each subscription, matching the * "persist now, not just at the top of this block" discipline already * used for the catchup and disabled-peer subscriptions: a crash @@ -5326,55 +5420,33 @@ create_reverse_subscriptions(BidirectionalState *state, const char *subscriber_n const char *dbname, const char *base_prov_connstr) { PQExpBuffer repsets = createPQExpBuffer(); - PQExpBuffer sub_name_buf = createPQExpBuffer(); + ReverseSubTarget *targets; + int num_targets; int i; printfPQExpBuffer(repsets, "{%s}", replication_sets); + targets = build_reverse_sub_targets(state, source_dsn, source_node_name, NULL, &num_targets); - for (i = 0; i < state->num_peers; i++) + for (i = 0; i < num_targets; i++) { - PeerNodeInfo *peer = &state->peers[i]; - PGconn *peer_conn; - char *reverse_sub_name; - - printfPQExpBuffer(sub_name_buf, "sub_%s_%s", peer->node_name, subscriber_name); - reverse_sub_name = pg_strdup(sub_name_buf->data); - - print_msg(VERBOSITY_NORMAL, - _("Creating reverse subscription \"%s\" on peer \"%s\"...\n"), - reverse_sub_name, peer->node_name); - peer_conn = connectdb(peer->dsn); - create_subscription_on_conn(peer_conn, reverse_sub_name, n3_dsn, - repsets->data, peer->node_name); - PQfinish(peer_conn); - - peer->reverse_sub_created = true; - write_manifest(state, subscriber_name, dbname, base_prov_connstr); - pg_free(reverse_sub_name); - } - - { - PGconn *source_conn; - char *reverse_sub_name; - - printfPQExpBuffer(sub_name_buf, "sub_%s_%s", source_node_name, subscriber_name); - reverse_sub_name = pg_strdup(sub_name_buf->data); + PGconn *conn; + char *reverse_sub_name = sub_name_for(targets[i].node_name, subscriber_name); print_msg(VERBOSITY_NORMAL, - _("Creating reverse subscription \"%s\" on the source...\n"), - reverse_sub_name); - source_conn = connectdb(source_dsn); - create_subscription_on_conn(source_conn, reverse_sub_name, n3_dsn, - repsets->data, "the source"); - PQfinish(source_conn); - - state->source_reverse_sub_created = true; + _("Creating reverse subscription \"%s\" on \"%s\"...\n"), + reverse_sub_name, targets[i].node_name); + conn = connectdb(targets[i].dsn); + create_subscription_on_conn(conn, reverse_sub_name, n3_dsn, + repsets->data, targets[i].node_name); + PQfinish(conn); + + *targets[i].reverse_sub_created = true; write_manifest(state, subscriber_name, dbname, base_prov_connstr); pg_free(reverse_sub_name); } + pg_free(targets); destroyPQExpBuffer(repsets); - destroyPQExpBuffer(sub_name_buf); } /* @@ -5389,42 +5461,25 @@ wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, const char *source_dsn, const char *source_node_name, const char *subscriber_name, int stall_timeout, int max_wait) { - PQExpBuffer sub_name_buf = createPQExpBuffer(); - PGconn *source_conn; + ReverseSubTarget *targets; PGresult *res; - int i; + int num_targets; int slot_count; + int i; - for (i = 0; i < state->num_peers; i++) - { - PeerNodeInfo *peer = &state->peers[i]; - PGconn *peer_conn; - char *reverse_sub_name; - - printfPQExpBuffer(sub_name_buf, "sub_%s_%s", peer->node_name, subscriber_name); - reverse_sub_name = pg_strdup(sub_name_buf->data); - - print_msg(VERBOSITY_NORMAL, - _("Waiting for reverse subscription \"%s\" on peer \"%s\"...\n"), - reverse_sub_name, peer->node_name); - peer_conn = connectdb(peer->dsn); - wait_for_sub_replicating(peer_conn, reverse_sub_name, stall_timeout, max_wait); - PQfinish(peer_conn); - pg_free(reverse_sub_name); - } + targets = build_reverse_sub_targets(state, source_dsn, source_node_name, NULL, &num_targets); + for (i = 0; i < num_targets; i++) { - char *reverse_sub_name; - - printfPQExpBuffer(sub_name_buf, "sub_%s_%s", source_node_name, subscriber_name); - reverse_sub_name = pg_strdup(sub_name_buf->data); + PGconn *conn; + char *reverse_sub_name = sub_name_for(targets[i].node_name, subscriber_name); print_msg(VERBOSITY_NORMAL, - _("Waiting for reverse subscription \"%s\" on the source...\n"), - reverse_sub_name); - source_conn = connectdb(source_dsn); - wait_for_sub_replicating(source_conn, reverse_sub_name, stall_timeout, max_wait); - PQfinish(source_conn); + _("Waiting for reverse subscription \"%s\" on \"%s\"...\n"), + reverse_sub_name, targets[i].node_name); + conn = connectdb(targets[i].dsn); + wait_for_sub_replicating(conn, reverse_sub_name, stall_timeout, max_wait); + PQfinish(conn); pg_free(reverse_sub_name); } @@ -5439,12 +5494,12 @@ wait_for_reverse_subs_ready(BidirectionalState *state, PGconn *n3_conn, } slot_count = atoi(PQgetvalue(res, 0, 0)); PQclear(res); - if (slot_count < state->num_peers + 1) + if (slot_count < num_targets) die(_("expected at least %d inbound replication slot(s) on n3 " "(one per peer plus the source), found %d\n"), - state->num_peers + 1, slot_count); + num_targets, slot_count); - destroyPQExpBuffer(sub_name_buf); + pg_free(targets); } /* @@ -5509,59 +5564,47 @@ verify_dataflow_to_n3(PGconn *n3_conn, const char *remote_node_name, static void verify_bidirectional_dataflow(BidirectionalState *state, PGconn *n3_conn, const char *source_dsn, const char *source_node_name, - const char *subscriber_name, + const char *source_sub_name, const char *subscriber_name, int stall_timeout, int max_wait) { + ReverseSubTarget *targets; char *n3_marker; + int num_targets; int i; + targets = build_reverse_sub_targets(state, source_dsn, source_node_name, + source_sub_name, &num_targets); + print_msg(VERBOSITY_NORMAL, _("Verifying n3's changes reach every peer and the source...\n")); n3_marker = get_sync_event_lsn(n3_conn, "n3"); - for (i = 0; i < state->num_peers; i++) + for (i = 0; i < num_targets; i++) { - PeerNodeInfo *peer = &state->peers[i]; - PGconn *peer_conn = connectdb(peer->dsn); + PGconn *conn = connectdb(targets[i].dsn); - verify_dataflow_from_n3(peer_conn, peer->node_name, subscriber_name, + verify_dataflow_from_n3(conn, targets[i].node_name, subscriber_name, n3_marker, stall_timeout, max_wait); - PQfinish(peer_conn); - } - - { - PGconn *source_conn = connectdb(source_dsn); - - verify_dataflow_from_n3(source_conn, source_node_name, subscriber_name, - n3_marker, stall_timeout, max_wait); - PQfinish(source_conn); + PQfinish(conn); } pg_free(n3_marker); print_msg(VERBOSITY_NORMAL, _("Verifying every peer's and the source's changes still reach " "n3...\n")); - for (i = 0; i < state->num_peers; i++) + for (i = 0; i < num_targets; i++) { - PeerNodeInfo *peer = &state->peers[i]; - PGconn *peer_conn = connectdb(peer->dsn); - char *peer_marker = get_sync_event_lsn(peer_conn, peer->node_name); + PGconn *conn = connectdb(targets[i].dsn); + char *remote_marker = get_sync_event_lsn(conn, targets[i].node_name); - PQfinish(peer_conn); - verify_dataflow_to_n3(n3_conn, peer->node_name, peer_marker, - peer->slot_name, peer->sub_name, stall_timeout, max_wait); - pg_free(peer_marker); + PQfinish(conn); + verify_dataflow_to_n3(n3_conn, targets[i].node_name, remote_marker, + targets[i].slot_name, targets[i].sub_name, + stall_timeout, max_wait); + pg_free(remote_marker); } - { - PGconn *source_conn = connectdb(source_dsn); - char *source_marker = get_sync_event_lsn(source_conn, source_node_name); - - PQfinish(source_conn); - verify_dataflow_to_n3(n3_conn, source_node_name, source_marker, - state->source_slot_name, NULL, stall_timeout, max_wait); - pg_free(source_marker); - } + pg_free(targets); } /* From c43949fdf889fd05e18096c1ab062740d0832e75 Mon Sep 17 00:00:00 2001 From: Asif Rehman Date: Thu, 27 Aug 2026 19:41:06 +0500 Subject: [PATCH 13/14] spock_create_subscriber: simplify and strengthen option parsing Add two shared option-parsing helpers and use them at every applicable getopt_long case: - parse_checked_int(arg, opt_name): strtol()-based integer parsing that dies on any non-numeric input, replacing --apply-delay's inline strtol block and --stall-timeout/--max-wait's bare atoi() (which silently parsed "abc" as 0 instead of rejecting it). - validated_existing_path(arg, file_label): expand_tilde() + file_exists() check, replacing four near-identical inline blocks for --postgresql-conf, --hba-conf, --recovery-conf, and --postgresql-auto-conf. Behavior is unchanged for valid input; --stall-timeout/--max-wait now reject non-numeric values instead of silently treating them as 0. --- tests/tap/t/048_bidir_join.pl | 42 ++++++++- .../spock_create_subscriber.c | 93 ++++++++++++------- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/tests/tap/t/048_bidir_join.pl b/tests/tap/t/048_bidir_join.pl index 544eee36d..770a93de4 100644 --- a/tests/tap/t/048_bidir_join.pl +++ b/tests/tap/t/048_bidir_join.pl @@ -65,6 +65,9 @@ # 1 n3 data directory removed after cleanup --force # 1 manifest removed after cleanup # 1 --bidirectional rejects a multi-database request +# 1 --max-wait rejects a value that overflows int +# 1 --stall-timeout rejects a non-numeric value +# 1 --apply-delay rejects a value that overflows int # 1 --bidirectional aborts when another database on the source has spock configured # 1 --bidirectional rejects a broken full-mesh topology (disabled subscription) # 1 --bidirectional rejects mismatched replication-set flags between source and peer @@ -84,12 +87,12 @@ # 1 pending sidecar removed once cleanup actually completed # 1 destroy_cluster # --- -# 69 total +# 72 total # ============================================================================= use strict; use warnings; -use Test::More tests => 69; +use Test::More tests => 72; use File::Path qw(remove_tree); use lib '.'; use SpockTest qw(create_cluster cross_wire destroy_cluster system_or_bail @@ -516,6 +519,41 @@ sub psql_capture { '--bidirectional rejects a multi-database request'); remove_tree($multidb_datadir) if -d $multidb_datadir; +# ============================================================================= +# TEST: option parsing rejects malformed/out-of-range integer arguments +# instead of silently truncating them -- e.g. a value like 4294967296 +# wraps to 0 when cast to int after strtol(), which would otherwise turn +# --max-wait=4294967296 into an accepted, effectively-unbounded wait. +# ============================================================================= +my $intcheck_datadir = '/tmp/tmp_spock_node_2_datadir_bidir_pr3_intcheck'; +remove_tree($intcheck_datadir) if -d $intcheck_datadir; +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--max-wait', '4294967296'), + '--max-wait rejects a value that overflows int'); + +ok(!system_maybe($SCS_BIN, + '--bidirectional', + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--stall-timeout', 'abc'), + '--stall-timeout rejects a non-numeric value'); + +ok(!system_maybe($SCS_BIN, + '--pgdata', $intcheck_datadir, + '--subscriber-name', 'n3', + '--provider-dsn', $n1_dsn, + '--subscriber-dsn', $n3_dsn, + '--apply-delay', '99999999999999999999'), + '--apply-delay rejects a value that overflows int'); +remove_tree($intcheck_datadir) if -d $intcheck_datadir; + # ============================================================================= # TEST: --bidirectional aborts if the source instance has spock configured # on another database too, even though that database was never named via diff --git a/utils/spock_create_subscriber/spock_create_subscriber.c b/utils/spock_create_subscriber/spock_create_subscriber.c index 8c4ac2fa6..fe93b3bbc 100644 --- a/utils/spock_create_subscriber/spock_create_subscriber.c +++ b/utils/spock_create_subscriber/spock_create_subscriber.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -199,6 +200,8 @@ static PGresult *debug_exec(PGconn *conn, const char *query); static int run_pg_ctl(const char *arg); static void validate_extra_basebackup_args(const char *args); +static int parse_checked_int(const char *arg, const char *opt_name); +static char *validated_existing_path(const char *arg, const char *file_label); static void run_basebackup(const char *provider_connstr, const char *data_dir, const char *extra_basebackup_args); static char *reset_subscriber_sysid(const char *data_dir); @@ -2338,26 +2341,14 @@ main(int argc, char **argv) replication_sets = validate_replication_set_input(pg_strdup(optarg)); break; case 4: - { - postgresql_conf = expand_tilde(pg_strdup(optarg)); - if (postgresql_conf != NULL && !file_exists(postgresql_conf)) - die(_("The specified postgresql.conf file does not exist.")); - break; - } + postgresql_conf = validated_existing_path(optarg, "postgresql.conf"); + break; case 5: - { - pg_hba_conf = expand_tilde(pg_strdup(optarg)); - if (pg_hba_conf != NULL && !file_exists(pg_hba_conf)) - die(_("The specified pg_hba.conf file does not exist.")); - break; - } + pg_hba_conf = validated_existing_path(optarg, "pg_hba.conf"); + break; case 6: - { - recovery_conf = expand_tilde(pg_strdup(optarg)); - if (recovery_conf != NULL && !file_exists(recovery_conf)) - die(_("The specified recovery configuration file does not exist.")); - break; - } + recovery_conf = validated_existing_path(optarg, "recovery configuration"); + break; case 'v': verbosity++; break; @@ -2368,12 +2359,7 @@ main(int argc, char **argv) drop_slot_if_exists = true; break; case 8: - { - char *endptr; - apply_delay = (int) strtol(optarg, &endptr, 10); - if (*endptr != '\0' || endptr == optarg) - die(_("--apply-delay requires an integer value\n")); - } + apply_delay = parse_checked_int(optarg, "apply-delay"); break; case 9: databases = pg_strdup(optarg); @@ -2389,12 +2375,12 @@ main(int argc, char **argv) bidir.enabled = true; break; case 13: - bidir.stall_timeout = atoi(optarg); + bidir.stall_timeout = parse_checked_int(optarg, "stall-timeout"); if (bidir.stall_timeout <= 0) die(_("--stall-timeout must be a positive integer")); break; case 14: - bidir.max_wait = atoi(optarg); + bidir.max_wait = parse_checked_int(optarg, "max-wait"); if (bidir.max_wait < 0) die(_("--max-wait must be a non-negative integer")); break; @@ -2405,12 +2391,8 @@ main(int argc, char **argv) bidir.force_cleanup = true; break; case 17: - { - postgresql_auto_conf = expand_tilde(pg_strdup(optarg)); - if (postgresql_auto_conf != NULL && !file_exists(postgresql_auto_conf)) - die(_("The specified postgresql.auto.conf file does not exist.")); - break; - } + postgresql_auto_conf = validated_existing_path(optarg, "postgresql.auto.conf"); + break; default: fprintf(stderr, _("Unknown option\n")); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); @@ -3340,6 +3322,53 @@ validate_extra_basebackup_args(const char *args) } } +/* + * Parse optarg as a base-10 integer, dying with a message naming + * opt_name if any part of it isn't numeric or the value doesn't fit in + * an int -- shared by every --