diff --git a/doc/pgprobackup.xml b/doc/pgprobackup.xml
index a347e7b43..76ec2cd76 100644
--- a/doc/pgprobackup.xml
+++ b/doc/pgprobackup.xml
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
backup_dir
instance_name
+ wal_file_path
wal_file_name
option
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b catchup_mode
Provides the path to the WAL file in
archive_command and
restore_command. Use the %p
- variable as the value for this option for correct processing.
+ variable as the value for this option or explicitly specify the path to a file
+ outside of the data directory. If you skip this option, the path
+ specified in pg_probackup.conf will be used.
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b catchup_mode
archive_command and
restore_command. Use the %f
variable as the value for this option for correct processing.
+ If the value of is a path
+ outside of the data directory, explicitly specify the filename.
diff --git a/src/archive.c b/src/archive.c
index 7bb8c1c03..0f32d9345 100644
--- a/src/archive.c
+++ b/src/archive.c
@@ -3,7 +3,7 @@
* archive.c: - pg_probackup specific archive commands for archive backups.
*
*
- * Portions Copyright (c) 2018-2019, Postgres Professional
+ * Portions Copyright (c) 2018-2021, Postgres Professional
*
*-------------------------------------------------------------------------
*/
@@ -113,15 +113,13 @@ static parray *setup_push_filelist(const char *archive_status_dir,
* Where archlog_path is $BACKUP_PATH/wal/instance_name
*/
void
-do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
+do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
char *wal_file_name, int batch_size, bool overwrite,
bool no_sync, bool no_ready_rename)
{
uint64 i;
- char current_dir[MAXPGPATH];
- char pg_xlog_dir[MAXPGPATH];
- char archive_status_dir[MAXPGPATH];
- uint64 system_id;
+ /* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
+ char archive_status_dir[MAXPGPATH] = "";
bool is_compress = false;
/* arrays with meta info for multi threaded backup */
@@ -141,31 +139,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
parray *batch_files = NULL;
int n_threads;
- if (wal_file_name == NULL)
- elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
-
- if (!getcwd(current_dir, sizeof(current_dir)))
- elog(ERROR, "getcwd() error");
-
- /* verify that archive-push --instance parameter is valid */
- system_id = get_system_identifier(current_dir, FIO_DB_HOST);
-
- if (instance->pgdata == NULL)
- elog(ERROR, "Cannot read pg_probackup.conf for this instance");
-
- if (system_id != instance->system_identifier)
- elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
- "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
- wal_file_name, instanceState->instance_name, instance->system_identifier, system_id);
-
- if (instance->compress_alg == PGLZ_COMPRESS)
- elog(ERROR, "Cannot use pglz for WAL compression");
-
- join_path_components(pg_xlog_dir, current_dir, XLOGDIR);
- join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
-
- /* Create 'archlog_path' directory. Do nothing if it already exists. */
- //fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
+ if (!no_ready_rename || batch_size > 1)
+ join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
#ifdef HAVE_LIBZ
if (instance->compress_alg == ZLIB_COMPRESS)
@@ -204,12 +179,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
{
int rc;
WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i);
+ bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0;
- rc = push_file(xlogfile, archive_status_dir,
+ rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir,
pg_xlog_dir, instanceState->instance_wal_subdir_path,
overwrite, no_sync,
instance->archive_timeout,
- no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false,
+ no_ready_rename || first_wal,
is_compress && IsXLogFileName(xlogfile->name) ? true : false,
instance->compress_level);
if (rc == 0)
@@ -233,7 +209,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
arg->first_filename = wal_file_name;
arg->archive_dir = instanceState->instance_wal_subdir_path;
arg->pg_xlog_dir = pg_xlog_dir;
- arg->archive_status_dir = archive_status_dir;
+ arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL;
arg->overwrite = overwrite;
arg->compress = is_compress;
arg->no_sync = no_sync;
@@ -276,7 +252,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
/* Note, that we are leaking memory here,
* because pushing into archive is a very
- * time-sensetive operation, so we skip freeing stuff.
+ * time-sensitive operation, so we skip freeing stuff.
*/
push_done:
@@ -356,9 +332,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
int compress_level)
{
int rc;
- char wal_file_dummy[MAXPGPATH];
-
- join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
elog(LOG, "pushing file \"%s\"", xlogfile->name);
@@ -375,11 +348,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
#endif
/* take '--no-ready-rename' flag into account */
- if (!no_ready_rename)
+ if (!no_ready_rename && archive_status_dir != NULL)
{
+ char wal_file_dummy[MAXPGPATH];
char wal_file_ready[MAXPGPATH];
char wal_file_done[MAXPGPATH];
+ join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready");
snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done");
diff --git a/src/backup.c b/src/backup.c
index 1d08c3828..c575865c4 100644
--- a/src/backup.c
+++ b/src/backup.c
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
uint64 system_id_conn;
uint64 system_id_pgdata;
- system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
+ system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
system_id_conn = get_remote_system_identifier(conn);
/* for checkdb check only system_id_pgdata and system_id_conn */
diff --git a/src/catchup.c b/src/catchup.c
index 5a0c8e45a..f9145a395 100644
--- a/src/catchup.c
+++ b/src/catchup.c
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons
/* Get WAL segments size and system ID of source PG instance */
instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata);
- instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST);
+ instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false);
current.start_time = time(NULL);
strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version));
@@ -163,7 +163,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
uint64 source_conn_id, source_id, dest_id;
source_conn_id = get_remote_system_identifier(source_conn);
- source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */
+ source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */
if (source_conn_id != source_id)
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
@@ -171,7 +171,7 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
if (current.backup_mode != BACKUP_MODE_FULL)
{
- dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST);
+ dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false);
if (source_conn_id != dest_id)
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
source_conn_id, dest_pgdata, dest_id);
diff --git a/src/help.c b/src/help.c
index 1515359e4..a6530fc0e 100644
--- a/src/help.c
+++ b/src/help.c
@@ -227,6 +227,7 @@ help_pg_probackup(void)
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" --wal-file-name=wal-file-name\n"));
+ printf(_(" [--wal-file-path=wal-file-path]\n"));
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
printf(_(" [--archive-timeout=timeout]\n"));
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
{
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
printf(_(" --wal-file-name=wal-file-name\n"));
+ printf(_(" [--wal-file-path=wal-file-path]\n"));
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
printf(_(" [--archive-timeout=timeout]\n"));
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
printf(_(" --instance=instance_name name of the instance to delete\n"));
printf(_(" --wal-file-name=wal-file-name\n"));
printf(_(" name of the file to copy into WAL archive\n"));
+ printf(_(" --wal-file-path=wal-file-path\n"));
+ printf(_(" relative destination path of the WAL archive\n"));
printf(_(" -j, --threads=NUM number of parallel threads\n"));
printf(_(" --batch-size=NUM number of files to be copied\n"));
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
help_archive_get(void)
{
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
- printf(_(" --wal-file-path=wal-file-path\n"));
printf(_(" --wal-file-name=wal-file-name\n"));
+ printf(_(" [--wal-file-path=wal-file-path]\n"));
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
printf(_(" [--no-validate-wal]\n"));
printf(_(" [--remote-proto] [--remote-host]\n"));
diff --git a/src/init.c b/src/init.c
index a4911cb5c..8773016b5 100644
--- a/src/init.c
+++ b/src/init.c
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
"(-D, --pgdata)");
/* Read system_identifier from PGDATA */
- instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST);
+ instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false);
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata);
diff --git a/src/pg_probackup.c b/src/pg_probackup.c
index d629d838d..49e226ace 100644
--- a/src/pg_probackup.c
+++ b/src/pg_probackup.c
@@ -35,7 +35,7 @@
* which includes info about pgdata directory and connection.
*
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
- * Portions Copyright (c) 2015-2019, Postgres Professional
+ * Portions Copyright (c) 2015-2021, Postgres Professional
*
*-------------------------------------------------------------------------
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
static char *wal_file_name;
static bool file_overwrite = false;
static bool no_ready_rename = false;
+static char archive_push_xlog_dir[MAXPGPATH] = "";
/* archive get options */
static char *prefetch_dir;
@@ -788,7 +789,7 @@ main(int argc, char *argv[])
current.stream = stream_wal = true;
if (instance_config.external_dir_str)
elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd));
- // TODO проверить instance_config.conn_opt
+ // TODO check instance_config.conn_opt
}
/* sanity */
@@ -796,6 +797,97 @@ main(int argc, char *argv[])
elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command",
get_subcmd_name(backup_subcmd));
+ if (backup_subcmd == ARCHIVE_PUSH_CMD)
+ {
+ /* Check archive-push parameters and construct archive_push_xlog_dir
+ *
+ * There are 4 cases:
+ * 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
+ * (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
+ * in this case we can use batches and threads
+ * 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
+ * in this case we can use this path, as well as batches and thread
+ * 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
+ * in this case we can use this path, as well as batches and thread
+ * (replica for example, see test_archive_push_sanity)
+ * 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
+ * disable optimizations and work with user specified path
+ */
+ bool check_system_id = true;
+ uint64 system_id;
+ char current_dir[MAXPGPATH];
+
+ if (wal_file_name == NULL)
+ elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
+
+ if (instance_config.pgdata == NULL)
+ elog(ERROR, "Cannot read pg_probackup.conf for this instance");
+
+ /* TODO may be remove in preference of checking inside compress_init()? */
+ if (instance_config.compress_alg == PGLZ_COMPRESS)
+ elog(ERROR, "Cannot use pglz for WAL compression");
+
+ if (!getcwd(current_dir, sizeof(current_dir)))
+ elog(ERROR, "getcwd() error");
+
+ if (wal_file_path == NULL)
+ {
+ /* 1st case */
+ system_id = get_system_identifier(current_dir, FIO_DB_HOST, false);
+ join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR);
+ }
+ else
+ {
+ /*
+ * Usually we get something like
+ * wal_file_path = "pg_wal/0000000100000000000000A1"
+ * wal_file_name = "0000000100000000000000A1"
+ * instance_config.pgdata = "/pgdata/.../node/data"
+ * We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
+ * and compare this directories.
+ * Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
+ */
+ char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
+ join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
+ if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST))
+ {
+ /* 2nd case */
+ system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false);
+ /* archive_push_xlog_dir already have right value */
+ }
+ else
+ {
+ if (strlen(stripped_wal_file_path) < MAXPGPATH)
+ strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH);
+ else
+ elog(ERROR, "Value specified to --wal_file_path is too long");
+
+ system_id = get_system_identifier(current_dir, FIO_DB_HOST, true);
+ /* 3rd case if control file present -- i.e. system_id != 0 */
+
+ if (system_id == 0)
+ {
+ /* 4th case */
+ check_system_id = false;
+
+ if (batch_size > 1 || num_threads > 1 || !no_ready_rename)
+ {
+ elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
+ batch_size = 1;
+ num_threads = 1;
+ no_ready_rename = true;
+ }
+ }
+ }
+ pfree(stripped_wal_file_path);
+ }
+
+ if (check_system_id && system_id != instance_config.system_identifier)
+ elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
+ "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
+ wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id);
+ }
+
#if PG_VERSION_NUM >= 100000
if (temp_slot && perm_slot)
elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -819,7 +911,7 @@ main(int argc, char *argv[])
switch (backup_subcmd)
{
case ARCHIVE_PUSH_CMD:
- do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name,
+ do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name,
batch_size, file_overwrite, no_sync, no_ready_rename);
break;
case ARCHIVE_GET_CMD:
diff --git a/src/pg_probackup.h b/src/pg_probackup.h
index 6a1feb014..a51794d98 100644
--- a/src/pg_probackup.h
+++ b/src/pg_probackup.h
@@ -889,7 +889,7 @@ extern int do_init(CatalogState *catalogState);
extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance);
/* in archive.c */
-extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
+extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
char *wal_file_name, int batch_size, bool overwrite,
bool no_sync, bool no_ready_rename);
extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path,
@@ -1153,7 +1153,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T
extern TimeLineID get_current_timeline(PGconn *conn);
extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe);
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
-extern uint64 get_system_identifier(const char *pgdata_path, fio_location location);
+extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe);
extern uint64 get_remote_system_identifier(PGconn *conn);
extern uint32 get_data_checksum_version(bool safe);
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
diff --git a/src/restore.c b/src/restore.c
index 005984aed..47e3b0344 100644
--- a/src/restore.c
+++ b/src/restore.c
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
*/
elog(INFO, "Trying to read pg_control file in destination directory");
- system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
+ system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
if (system_id_pgdata == instance_config.system_identifier)
system_id_match = true;
diff --git a/src/util.c b/src/util.c
index f39b31d45..fb33fd046 100644
--- a/src/util.c
+++ b/src/util.c
@@ -247,15 +247,15 @@ get_checkpoint_location(PGconn *conn)
}
uint64
-get_system_identifier(const char *pgdata_path, fio_location location)
+get_system_identifier(const char *pgdata_path, fio_location location, bool safe)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
/* First fetch file... */
- buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, location);
- if (buffer == NULL)
+ buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location);
+ if (safe && buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
diff --git a/src/utils/file.c b/src/utils/file.c
index 810b4b394..7d1df554b 100644
--- a/src/utils/file.c
+++ b/src/utils/file.c
@@ -1141,6 +1141,33 @@ fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location lo
}
}
+/*
+ * Compare, that filename1 and filename2 is the same file
+ * in windows compare only filenames
+ */
+bool
+fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location)
+{
+#ifndef WIN32
+ struct stat stat1, stat2;
+
+ if (fio_stat(filename1, &stat1, follow_symlink, location) < 0)
+ elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno));
+
+ if (fio_stat(filename2, &stat2, follow_symlink, location) < 0)
+ elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno));
+
+ return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev;
+#else
+ char *abs_name1 = make_absolute_path(filename1);
+ char *abs_name2 = make_absolute_path(filename2);
+ bool result = strcmp(abs_name1, abs_name2) == 0;
+ free(abs_name2);
+ free(abs_name1);
+ return result;
+#endif
+}
+
/*
* Read value of a symbolic link
* this is a wrapper about readlink() syscall
diff --git a/src/utils/file.h b/src/utils/file.h
index edb5ea0f9..a554b4ab0 100644
--- a/src/utils/file.h
+++ b/src/utils/file.h
@@ -129,6 +129,7 @@ extern int fio_mkdir(char const* path, int mode, fio_location location);
extern int fio_chmod(char const* path, int mode, fio_location location);
extern int fio_access(char const* path, int mode, fio_location location);
extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location);
+extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location);
extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location);
extern DIR* fio_opendir(char const* path, fio_location location);
extern struct dirent * fio_readdir(DIR *dirp);
diff --git a/src/utils/pgut.c b/src/utils/pgut.c
index 52599848d..2cf0ccbe7 100644
--- a/src/utils/pgut.c
+++ b/src/utils/pgut.c
@@ -977,6 +977,22 @@ pgut_strndup(const char *str, size_t n)
return ret;
}
+/*
+ * Allocates new string, that contains part of filepath string minus trailing filename string
+ * If trailing filename string not found, returns copy of filepath.
+ * Result must be free by caller.
+ */
+char *
+pgut_str_strip_trailing_filename(const char *filepath, const char *filename)
+{
+ size_t fp_len = strlen(filepath);
+ size_t fn_len = strlen(filename);
+ if (strncmp(filepath + fp_len - fn_len, filename, fn_len) == 0)
+ return pgut_strndup(filepath, fp_len - fn_len);
+ else
+ return pgut_strndup(filepath, fp_len);
+}
+
FILE *
pgut_fopen(const char *path, const char *mode, bool missing_ok)
{
diff --git a/src/utils/pgut.h b/src/utils/pgut.h
index a1d7b5a93..fa0efe816 100644
--- a/src/utils/pgut.h
+++ b/src/utils/pgut.h
@@ -63,6 +63,7 @@ extern void *pgut_malloc0(size_t size);
extern void *pgut_realloc(void *p, size_t size);
extern char *pgut_strdup(const char *str);
extern char *pgut_strndup(const char *str, size_t n);
+extern char *pgut_str_strip_trailing_filename(const char *filepath, const char *filename);
#define pgut_new(type) ((type *) pgut_malloc(sizeof(type)))
#define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type)))
diff --git a/tests/archive.py b/tests/archive.py
index 5157e8b89..22b9d8693 100644
--- a/tests/archive.py
+++ b/tests/archive.py
@@ -1828,6 +1828,133 @@ def test_archive_options_1(self):
self.del_test_dir(module_name, fname)
+ # @unittest.skip("skip")
+ # @unittest.expectedFailure
+ def test_undefined_wal_file_path(self):
+ """
+ check that archive-push works correct with undefined
+ --wal-file-path
+ """
+ fname = self.id().split('.')[3]
+ backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+ node = self.make_simple_node(
+ base_dir=os.path.join(module_name, fname, 'node'),
+ set_replication=True,
+ initdb_params=['--data-checksums'])
+
+ self.init_pb(backup_dir)
+ self.add_instance(backup_dir, 'node', node)
+ self.set_archiving(backup_dir, 'node', node)
+ if os.name == 'posix':
+ archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format(
+ self.probackup_path, backup_dir, 'node')
+ elif os.name == 'nt':
+ archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format(
+ self.probackup_path, backup_dir, 'node').replace("\\","\\\\")
+ else:
+ self.assertTrue(False, 'Unexpected os family')
+
+ self.set_auto_conf(
+ node,
+ {'archive_command': archive_command})
+
+ node.slow_start()
+ node.safe_psql(
+ "postgres",
+ "create table t_heap as select i"
+ " as id from generate_series(0, 10) i")
+ self.switch_wal_segment(node)
+
+ # check
+ self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001')
+
+ self.del_test_dir(module_name, fname)
+
+ # @unittest.skip("skip")
+ # @unittest.expectedFailure
+ def test_intermediate_archiving(self):
+ """
+ check that archive-push works correct with --wal-file-path setting by user
+ """
+ fname = self.id().split('.')[3]
+ backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+
+ node = self.make_simple_node(
+ base_dir=os.path.join(module_name, fname, 'node'),
+ initdb_params=['--data-checksums'])
+
+ node_pg_options = {}
+ if node.major_version >= 13:
+ node_pg_options['wal_keep_size'] = '0MB'
+ else:
+ node_pg_options['wal_keep_segments'] = '0'
+ self.set_auto_conf(node, node_pg_options)
+
+ self.init_pb(backup_dir)
+ self.add_instance(backup_dir, 'node', node)
+
+ wal_dir = os.path.join(self.tmp_path, module_name, fname, 'intermediate_dir')
+ shutil.rmtree(wal_dir, ignore_errors=True)
+ os.makedirs(wal_dir)
+ if os.name == 'posix':
+ self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir))
+ elif os.name == 'nt':
+ self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\")))
+ else:
+ self.assertTrue(False, 'Unexpected os family')
+
+ node.slow_start()
+ node.safe_psql(
+ "postgres",
+ "create table t_heap as select i"
+ " as id from generate_series(0, 10) i")
+ self.switch_wal_segment(node)
+
+ wal_segment = '000000010000000000000001'
+
+ self.run_pb(["archive-push", "-B", backup_dir,
+ "--instance=node", "-D", node.data_dir,
+ "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment])
+
+ self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment)
+
+ self.del_test_dir(module_name, fname)
+
+ # @unittest.skip("skip")
+ # @unittest.expectedFailure
+ def test_waldir_outside_pgdata_archiving(self):
+ """
+ check that archive-push works correct with symlinked waldir
+ """
+ if self.pg_config_version < self.version_to_num('10.0'):
+ return unittest.skip(
+ 'Skipped because waldir outside pgdata is supported since PG 10')
+
+ fname = self.id().split('.')[3]
+ backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+ external_wal_dir = os.path.join(self.tmp_path, module_name, fname, 'ext_wal_dir')
+ shutil.rmtree(external_wal_dir, ignore_errors=True)
+
+ node = self.make_simple_node(
+ base_dir=os.path.join(module_name, fname, 'node'),
+ initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)])
+
+ self.init_pb(backup_dir)
+ self.add_instance(backup_dir, 'node', node)
+ self.set_archiving(backup_dir, 'node', node)
+
+ node.slow_start()
+ node.safe_psql(
+ "postgres",
+ "create table t_heap as select i"
+ " as id from generate_series(0, 10) i")
+ self.switch_wal_segment(node)
+
+ # check
+ self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001')
+
+ self.del_test_dir(module_name, fname)
+
# @unittest.skip("skip")
# @unittest.expectedFailure
def test_hexadecimal_timeline(self):
diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out
index 01384a893..dd3c4e865 100644
--- a/tests/expected/option_help.out
+++ b/tests/expected/option_help.out
@@ -144,6 +144,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database.
pg_probackup archive-push -B backup-path --instance=instance_name
--wal-file-name=wal-file-name
+ [--wal-file-path=wal-file-path]
[-j num-threads] [--batch-size=batch_size]
[--archive-timeout=timeout]
[--no-ready-rename] [--no-sync]
diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py
index 1b54d3165..3b14b7170 100644
--- a/tests/helpers/ptrack_helpers.py
+++ b/tests/helpers/ptrack_helpers.py
@@ -1296,7 +1296,8 @@ def get_recovery_conf(self, node):
def set_archiving(
self, backup_dir, instance, node, replica=False,
overwrite=False, compress=True, old_binary=False,
- log_level=False, archive_timeout=False):
+ log_level=False, archive_timeout=False,
+ custom_archive_command=None):
# parse postgresql.auto.conf
options = {}
@@ -1306,45 +1307,47 @@ def set_archiving(
else:
options['archive_mode'] = 'on'
- if os.name == 'posix':
- options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format(
- self.probackup_path, backup_dir, instance)
-
- elif os.name == 'nt':
- options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format(
- self.probackup_path.replace("\\","\\\\"),
- backup_dir.replace("\\","\\\\"), instance)
+ if custom_archive_command is None:
+ if os.name == 'posix':
+ options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format(
+ self.probackup_path, backup_dir, instance)
- # don`t forget to kill old_binary after remote ssh release
- if self.remote and not old_binary:
- options['archive_command'] += '--remote-proto=ssh '
- options['archive_command'] += '--remote-host=localhost '
+ elif os.name == 'nt':
+ options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format(
+ self.probackup_path.replace("\\","\\\\"),
+ backup_dir.replace("\\","\\\\"), instance)
- if self.archive_compress and compress:
- options['archive_command'] += '--compress '
+ # don`t forget to kill old_binary after remote ssh release
+ if self.remote and not old_binary:
+ options['archive_command'] += '--remote-proto=ssh '
+ options['archive_command'] += '--remote-host=localhost '
- if overwrite:
- options['archive_command'] += '--overwrite '
+ if self.archive_compress and compress:
+ options['archive_command'] += '--compress '
- options['archive_command'] += '--log-level-console=VERBOSE '
- options['archive_command'] += '-j 5 '
- options['archive_command'] += '--batch-size 10 '
- options['archive_command'] += '--no-sync '
+ if overwrite:
+ options['archive_command'] += '--overwrite '
- if archive_timeout:
- options['archive_command'] += '--archive-timeout={0} '.format(
- archive_timeout)
+ options['archive_command'] += '--log-level-console=VERBOSE '
+ options['archive_command'] += '-j 5 '
+ options['archive_command'] += '--batch-size 10 '
+ options['archive_command'] += '--no-sync '
- if os.name == 'posix':
- options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f'
+ if archive_timeout:
+ options['archive_command'] += '--archive-timeout={0} '.format(
+ archive_timeout)
- elif os.name == 'nt':
- options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"'
+ if os.name == 'posix':
+ options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f'
- if log_level:
- options['archive_command'] += ' --log-level-console={0}'.format(log_level)
- options['archive_command'] += ' --log-level-file={0} '.format(log_level)
+ elif os.name == 'nt':
+ options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"'
+ if log_level:
+ options['archive_command'] += ' --log-level-console={0}'.format(log_level)
+ options['archive_command'] += ' --log-level-file={0} '.format(log_level)
+ else: # custom_archive_command is not None
+ options['archive_command'] = custom_archive_command
self.set_auto_conf(node, options)