Skip to content

Commit b87ca18

Browse files
dlepikhovakulaginmindrups
authored
[Issue #265][PGPRO-5421] archive-push backward compatibility (#437)
Restore the --wal-file-path option of the archive-push command (it was ignored since a196073) Co-authored-by: Mikhail A. Kulagin <[email protected]> Co-authored-by: Elena Indrupskaya <[email protected]>
1 parent 5b6ca62 commit b87ca18

17 files changed

+338
-86
lines changed

Diff for: ‎doc/pgprobackup.xml

+6-1
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ doc/src/sgml/pgprobackup.sgml
131131
<arg choice="plain"><option>archive-push</option></arg>
132132
<arg choice="plain"><option>-B</option> <replaceable>backup_dir</replaceable></arg>
133133
<arg choice="plain"><option>--instance</option> <replaceable>instance_name</replaceable></arg>
134+
<arg choice="plain"><option>--wal-file-path</option> <replaceable>wal_file_path</replaceable></arg>
134135
<arg choice="plain"><option>--wal-file-name</option> <replaceable>wal_file_name</replaceable></arg>
135136
<arg rep="repeat"><replaceable>option</replaceable></arg>
136137
</cmdsynopsis>
@@ -5367,7 +5368,9 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53675368
Provides the path to the WAL file in
53685369
<parameter>archive_command</parameter> and
53695370
<parameter>restore_command</parameter>. Use the <literal>%p</literal>
5370-
variable as the value for this option for correct processing.
5371+
variable as the value for this option or explicitly specify the path to a file
5372+
outside of the data directory. If you skip this option, the path
5373+
specified in <filename>pg_probackup.conf</filename> will be used.
53715374
</para>
53725375
</listitem>
53735376
</varlistentry>
@@ -5380,6 +5383,8 @@ pg_probackup catchup -b <replaceable>catchup_mode</replaceable>
53805383
<parameter>archive_command</parameter> and
53815384
<parameter>restore_command</parameter>. Use the <literal>%f</literal>
53825385
variable as the value for this option for correct processing.
5386+
If the value of <option>--wal-file-path</option> is a path
5387+
outside of the data directory, explicitly specify the filename.
53835388
</para>
53845389
</listitem>
53855390
</varlistentry>

Diff for: ‎src/archive.c

+14-39
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* archive.c: - pg_probackup specific archive commands for archive backups.
44
*
55
*
6-
* Portions Copyright (c) 2018-2019, Postgres Professional
6+
* Portions Copyright (c) 2018-2021, Postgres Professional
77
*
88
*-------------------------------------------------------------------------
99
*/
@@ -113,15 +113,13 @@ static parray *setup_push_filelist(const char *archive_status_dir,
113113
* Where archlog_path is $BACKUP_PATH/wal/instance_name
114114
*/
115115
void
116-
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
116+
do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
117117
char *wal_file_name, int batch_size, bool overwrite,
118118
bool no_sync, bool no_ready_rename)
119119
{
120120
uint64 i;
121-
char current_dir[MAXPGPATH];
122-
char pg_xlog_dir[MAXPGPATH];
123-
char archive_status_dir[MAXPGPATH];
124-
uint64 system_id;
121+
/* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */
122+
char archive_status_dir[MAXPGPATH] = "";
125123
bool is_compress = false;
126124

127125
/* arrays with meta info for multi threaded backup */
@@ -141,31 +139,8 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
141139
parray *batch_files = NULL;
142140
int n_threads;
143141

144-
if (wal_file_name == NULL)
145-
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
146-
147-
if (!getcwd(current_dir, sizeof(current_dir)))
148-
elog(ERROR, "getcwd() error");
149-
150-
/* verify that archive-push --instance parameter is valid */
151-
system_id = get_system_identifier(current_dir, FIO_DB_HOST);
152-
153-
if (instance->pgdata == NULL)
154-
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
155-
156-
if (system_id != instance->system_identifier)
157-
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
158-
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
159-
wal_file_name, instanceState->instance_name, instance->system_identifier, system_id);
160-
161-
if (instance->compress_alg == PGLZ_COMPRESS)
162-
elog(ERROR, "Cannot use pglz for WAL compression");
163-
164-
join_path_components(pg_xlog_dir, current_dir, XLOGDIR);
165-
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
166-
167-
/* Create 'archlog_path' directory. Do nothing if it already exists. */
168-
//fio_mkdir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, FIO_BACKUP_HOST);
142+
if (!no_ready_rename || batch_size > 1)
143+
join_path_components(archive_status_dir, pg_xlog_dir, "archive_status");
169144

170145
#ifdef HAVE_LIBZ
171146
if (instance->compress_alg == ZLIB_COMPRESS)
@@ -204,12 +179,13 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
204179
{
205180
int rc;
206181
WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i);
182+
bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0;
207183

208-
rc = push_file(xlogfile, archive_status_dir,
184+
rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir,
209185
pg_xlog_dir, instanceState->instance_wal_subdir_path,
210186
overwrite, no_sync,
211187
instance->archive_timeout,
212-
no_ready_rename || (strcmp(xlogfile->name, wal_file_name) == 0) ? true : false,
188+
no_ready_rename || first_wal,
213189
is_compress && IsXLogFileName(xlogfile->name) ? true : false,
214190
instance->compress_level);
215191
if (rc == 0)
@@ -233,7 +209,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
233209
arg->first_filename = wal_file_name;
234210
arg->archive_dir = instanceState->instance_wal_subdir_path;
235211
arg->pg_xlog_dir = pg_xlog_dir;
236-
arg->archive_status_dir = archive_status_dir;
212+
arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL;
237213
arg->overwrite = overwrite;
238214
arg->compress = is_compress;
239215
arg->no_sync = no_sync;
@@ -276,7 +252,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
276252

277253
/* Note, that we are leaking memory here,
278254
* because pushing into archive is a very
279-
* time-sensetive operation, so we skip freeing stuff.
255+
* time-sensitive operation, so we skip freeing stuff.
280256
*/
281257

282258
push_done:
@@ -356,9 +332,6 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
356332
int compress_level)
357333
{
358334
int rc;
359-
char wal_file_dummy[MAXPGPATH];
360-
361-
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
362335

363336
elog(LOG, "pushing file \"%s\"", xlogfile->name);
364337

@@ -375,11 +348,13 @@ push_file(WALSegno *xlogfile, const char *archive_status_dir,
375348
#endif
376349

377350
/* take '--no-ready-rename' flag into account */
378-
if (!no_ready_rename)
351+
if (!no_ready_rename && archive_status_dir != NULL)
379352
{
353+
char wal_file_dummy[MAXPGPATH];
380354
char wal_file_ready[MAXPGPATH];
381355
char wal_file_done[MAXPGPATH];
382356

357+
join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name);
383358
snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready");
384359
snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done");
385360

Diff for: ‎src/backup.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -943,7 +943,7 @@ check_system_identifiers(PGconn *conn, const char *pgdata)
943943
uint64 system_id_conn;
944944
uint64 system_id_pgdata;
945945

946-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
946+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
947947
system_id_conn = get_remote_system_identifier(conn);
948948

949949
/* for checkdb check only system_id_pgdata and system_id_conn */

Diff for: ‎src/catchup.c

+3-3
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, cons
4848

4949
/* Get WAL segments size and system ID of source PG instance */
5050
instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata);
51-
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST);
51+
instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false);
5252
current.start_time = time(NULL);
5353

5454
strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version));
@@ -163,15 +163,15 @@ catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn,
163163
uint64 source_conn_id, source_id, dest_id;
164164

165165
source_conn_id = get_remote_system_identifier(source_conn);
166-
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST); /* same as instance_config.system_identifier */
166+
source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */
167167

168168
if (source_conn_id != source_id)
169169
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
170170
source_conn_id, source_pgdata, source_id);
171171

172172
if (current.backup_mode != BACKUP_MODE_FULL)
173173
{
174-
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST);
174+
dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false);
175175
if (source_conn_id != dest_id)
176176
elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu",
177177
source_conn_id, dest_pgdata, dest_id);

Diff for: ‎src/help.c

+5-1
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ help_pg_probackup(void)
227227

228228
printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
229229
printf(_(" --wal-file-name=wal-file-name\n"));
230+
printf(_(" [--wal-file-path=wal-file-path]\n"));
230231
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
231232
printf(_(" [--archive-timeout=timeout]\n"));
232233
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -937,6 +938,7 @@ help_archive_push(void)
937938
{
938939
printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
939940
printf(_(" --wal-file-name=wal-file-name\n"));
941+
printf(_(" [--wal-file-path=wal-file-path]\n"));
940942
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
941943
printf(_(" [--archive-timeout=timeout]\n"));
942944
printf(_(" [--no-ready-rename] [--no-sync]\n"));
@@ -951,6 +953,8 @@ help_archive_push(void)
951953
printf(_(" --instance=instance_name name of the instance to delete\n"));
952954
printf(_(" --wal-file-name=wal-file-name\n"));
953955
printf(_(" name of the file to copy into WAL archive\n"));
956+
printf(_(" --wal-file-path=wal-file-path\n"));
957+
printf(_(" relative destination path of the WAL archive\n"));
954958
printf(_(" -j, --threads=NUM number of parallel threads\n"));
955959
printf(_(" --batch-size=NUM number of files to be copied\n"));
956960
printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n"));
@@ -981,8 +985,8 @@ static void
981985
help_archive_get(void)
982986
{
983987
printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME);
984-
printf(_(" --wal-file-path=wal-file-path\n"));
985988
printf(_(" --wal-file-name=wal-file-name\n"));
989+
printf(_(" [--wal-file-path=wal-file-path]\n"));
986990
printf(_(" [-j num-threads] [--batch-size=batch_size]\n"));
987991
printf(_(" [--no-validate-wal]\n"));
988992
printf(_(" [--remote-proto] [--remote-host]\n"));

Diff for: ‎src/init.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
5757
"(-D, --pgdata)");
5858

5959
/* Read system_identifier from PGDATA */
60-
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST);
60+
instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false);
6161
/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
6262
instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata);
6363

Diff for: ‎src/pg_probackup.c

+95-3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
* which includes info about pgdata directory and connection.
3636
*
3737
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
38-
* Portions Copyright (c) 2015-2019, Postgres Professional
38+
* Portions Copyright (c) 2015-2021, Postgres Professional
3939
*
4040
*-------------------------------------------------------------------------
4141
*/
@@ -151,6 +151,7 @@ static char *wal_file_path;
151151
static char *wal_file_name;
152152
static bool file_overwrite = false;
153153
static bool no_ready_rename = false;
154+
static char archive_push_xlog_dir[MAXPGPATH] = "";
154155

155156
/* archive get options */
156157
static char *prefetch_dir;
@@ -788,14 +789,105 @@ main(int argc, char *argv[])
788789
current.stream = stream_wal = true;
789790
if (instance_config.external_dir_str)
790791
elog(ERROR, "external directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd));
791-
// TODO проверить instance_config.conn_opt
792+
// TODO check instance_config.conn_opt
792793
}
793794

794795
/* sanity */
795796
if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate)
796797
elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command",
797798
get_subcmd_name(backup_subcmd));
798799

800+
if (backup_subcmd == ARCHIVE_PUSH_CMD)
801+
{
802+
/* Check archive-push parameters and construct archive_push_xlog_dir
803+
*
804+
* There are 4 cases:
805+
* 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files
806+
* (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push())
807+
* in this case we can use batches and threads
808+
* 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata)
809+
* in this case we can use this path, as well as batches and thread
810+
* 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id
811+
* in this case we can use this path, as well as batches and thread
812+
* (replica for example, see test_archive_push_sanity)
813+
* 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found
814+
* disable optimizations and work with user specified path
815+
*/
816+
bool check_system_id = true;
817+
uint64 system_id;
818+
char current_dir[MAXPGPATH];
819+
820+
if (wal_file_name == NULL)
821+
elog(ERROR, "Required parameter is not specified: --wal-file-name %%f");
822+
823+
if (instance_config.pgdata == NULL)
824+
elog(ERROR, "Cannot read pg_probackup.conf for this instance");
825+
826+
/* TODO may be remove in preference of checking inside compress_init()? */
827+
if (instance_config.compress_alg == PGLZ_COMPRESS)
828+
elog(ERROR, "Cannot use pglz for WAL compression");
829+
830+
if (!getcwd(current_dir, sizeof(current_dir)))
831+
elog(ERROR, "getcwd() error");
832+
833+
if (wal_file_path == NULL)
834+
{
835+
/* 1st case */
836+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, false);
837+
join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR);
838+
}
839+
else
840+
{
841+
/*
842+
* Usually we get something like
843+
* wal_file_path = "pg_wal/0000000100000000000000A1"
844+
* wal_file_name = "0000000100000000000000A1"
845+
* instance_config.pgdata = "/pgdata/.../node/data"
846+
* We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata
847+
* and compare this directories.
848+
* Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving)
849+
*/
850+
char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
851+
join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
852+
if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST))
853+
{
854+
/* 2nd case */
855+
system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false);
856+
/* archive_push_xlog_dir already have right value */
857+
}
858+
else
859+
{
860+
if (strlen(stripped_wal_file_path) < MAXPGPATH)
861+
strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH);
862+
else
863+
elog(ERROR, "Value specified to --wal_file_path is too long");
864+
865+
system_id = get_system_identifier(current_dir, FIO_DB_HOST, true);
866+
/* 3rd case if control file present -- i.e. system_id != 0 */
867+
868+
if (system_id == 0)
869+
{
870+
/* 4th case */
871+
check_system_id = false;
872+
873+
if (batch_size > 1 || num_threads > 1 || !no_ready_rename)
874+
{
875+
elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename");
876+
batch_size = 1;
877+
num_threads = 1;
878+
no_ready_rename = true;
879+
}
880+
}
881+
}
882+
pfree(stripped_wal_file_path);
883+
}
884+
885+
if (check_system_id && system_id != instance_config.system_identifier)
886+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
887+
"Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT,
888+
wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id);
889+
}
890+
799891
#if PG_VERSION_NUM >= 100000
800892
if (temp_slot && perm_slot)
801893
elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option");
@@ -819,7 +911,7 @@ main(int argc, char *argv[])
819911
switch (backup_subcmd)
820912
{
821913
case ARCHIVE_PUSH_CMD:
822-
do_archive_push(instanceState, &instance_config, wal_file_path, wal_file_name,
914+
do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name,
823915
batch_size, file_overwrite, no_sync, no_ready_rename);
824916
break;
825917
case ARCHIVE_GET_CMD:

Diff for: ‎src/pg_probackup.h

+2-2
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,7 @@ extern int do_init(CatalogState *catalogState);
889889
extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance);
890890

891891
/* in archive.c */
892-
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
892+
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir,
893893
char *wal_file_name, int batch_size, bool overwrite,
894894
bool no_sync, bool no_ready_rename);
895895
extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path,
@@ -1153,7 +1153,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, T
11531153
extern TimeLineID get_current_timeline(PGconn *conn);
11541154
extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe);
11551155
extern XLogRecPtr get_checkpoint_location(PGconn *conn);
1156-
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location);
1156+
extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe);
11571157
extern uint64 get_remote_system_identifier(PGconn *conn);
11581158
extern uint32 get_data_checksum_version(bool safe);
11591159
extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);

Diff for: ‎src/restore.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -2186,7 +2186,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
21862186
*/
21872187
elog(INFO, "Trying to read pg_control file in destination directory");
21882188

2189-
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST);
2189+
system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false);
21902190

21912191
if (system_id_pgdata == instance_config.system_identifier)
21922192
system_id_match = true;

0 commit comments

Comments
 (0)