Merge branch 'master' into pgpro-2071

Author: za-arthur

Diff for: src/backup.c

@@ -24,8 +24,6 @@
 #include "utils/thread.h"
 #include <time.h>
 
-#define PG_STOP_BACKUP_TIMEOUT 300
-
 /*
  * Macro needed to parse ptrack.
  * NOTE Keep those values syncronised with definitions in ptrack.h
@@ -310,7 +308,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
 			to_path, strerror(errno_tmp));
 	}
 
-	INIT_TRADITIONAL_CRC32(file->crc);
+	INIT_FILE_CRC32(true, file->crc);
 
 	/* read from stream and write to backup file */
 	while (1)
@@ -336,14 +334,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
 		{
 			write_buffer_size = Min(row_length, sizeof(buf));
 			memcpy(buf, copybuf, write_buffer_size);
-			COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size);
+			COMP_FILE_CRC32(true, file->crc, buf, write_buffer_size);
 
 			/* TODO calc checksum*/
 			if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
 			{
 				errno_tmp = errno;
 				/* oops */
-				FIN_TRADITIONAL_CRC32(file->crc);
+				FIN_FILE_CRC32(true, file->crc);
 				fclose(out);
 				PQfinish(conn);
 				elog(ERROR, "cannot write to \"%s\": %s", to_path,
@@ -367,7 +365,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
 	}
 
 	file->write_size = (int64) file->read_size;
-	FIN_TRADITIONAL_CRC32(file->crc);
+	FIN_FILE_CRC32(true, file->crc);
 
 	fclose(out);
 }
@@ -1883,8 +1881,8 @@ pg_stop_backup(pgBackup *backup)
 	}
 
 	/*
-	 * Wait for the result of pg_stop_backup(),
-	 * but no longer than PG_STOP_BACKUP_TIMEOUT seconds
+	 * Wait for the result of pg_stop_backup(), but no longer than
+	 * archive_timeout seconds
 	 */
 	if (pg_stop_backup_is_sent && !in_cleanup)
 	{
@@ -1907,14 +1905,14 @@ pg_stop_backup(pgBackup *backup)
 					elog(INFO, "wait for pg_stop_backup()");
 
 				/*
-				 * If postgres haven't answered in PG_STOP_BACKUP_TIMEOUT seconds,
+				 * If postgres haven't answered in archive_timeout seconds,
 				 * send an interrupt.
 				 */
-				if (pg_stop_backup_timeout > PG_STOP_BACKUP_TIMEOUT)
+				if (pg_stop_backup_timeout > instance_config.archive_timeout)
 				{
 					pgut_cancel(conn);
 					elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it",
-						 PG_STOP_BACKUP_TIMEOUT);
+						 instance_config.archive_timeout);
 				}
 			}
 			else
@@ -2290,9 +2288,12 @@ backup_files(void *arg)
 					continue;
 				}
 			}
+			else if (strcmp(file->name, "pg_control") == 0)
+				copy_pgcontrol_file(arguments->from_root, arguments->to_root,
+									file);
 			else
 			{
-				bool skip = false;
+				bool		skip = false;
 
 				/* If non-data file has not changed since last backup... */
 				if (prev_file && file->exists_in_prev &&

Diff for: src/data.c

@@ -487,7 +487,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
 				  blknum, header.compressed_size, write_buffer_size); */
 
 	/* Update CRC */
-	COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
+	COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size);
 
 	/* write data page */
 	if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
@@ -547,13 +547,13 @@ backup_data_file(backup_files_arg* arguments,
 	/* reset size summary */
 	file->read_size = 0;
 	file->write_size = 0;
-	INIT_TRADITIONAL_CRC32(file->crc);
+	INIT_FILE_CRC32(true, file->crc);
 
 	/* open backup mode file for read */
 	in = fopen(file->path, PG_BINARY_R);
 	if (in == NULL)
 	{
-		FIN_TRADITIONAL_CRC32(file->crc);
+		FIN_FILE_CRC32(true, file->crc);
 
 		/*
 		 * If file is not found, this is not en error.
@@ -658,7 +658,7 @@ backup_data_file(backup_files_arg* arguments,
 			 to_path, strerror(errno));
 	fclose(in);
 
-	FIN_TRADITIONAL_CRC32(file->crc);
+	FIN_FILE_CRC32(true, file->crc);
 
 	/*
 	 * If we have pagemap then file in the backup can't be a zero size.
@@ -927,7 +927,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
 	struct stat	st;
 	pg_crc32	crc;
 
-	INIT_TRADITIONAL_CRC32(crc);
+	INIT_FILE_CRC32(true, crc);
 
 	/* reset size summary */
 	file->read_size = 0;
@@ -937,7 +937,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
 	in = fopen(file->path, PG_BINARY_R);
 	if (in == NULL)
 	{
-		FIN_TRADITIONAL_CRC32(crc);
+		FIN_FILE_CRC32(true, crc);
 		file->crc = crc;
 
 		/* maybe deleted, it's not error */
@@ -986,7 +986,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
 				 strerror(errno_tmp));
 		}
 		/* update CRC */
-		COMP_TRADITIONAL_CRC32(crc, buf, read_len);
+		COMP_FILE_CRC32(true, crc, buf, read_len);
 
 		file->read_size += read_len;
 	}
@@ -1013,14 +1013,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
 				 strerror(errno_tmp));
 		}
 		/* update CRC */
-		COMP_TRADITIONAL_CRC32(crc, buf, read_len);
+		COMP_FILE_CRC32(true, crc, buf, read_len);
 
 		file->read_size += read_len;
 	}
 
 	file->write_size = (int64) file->read_size;
 	/* finish CRC calculation and store into pgFile */
-	FIN_TRADITIONAL_CRC32(crc);
+	FIN_FILE_CRC32(true, crc);
 	file->crc = crc;
 
 	/* update file permission */
@@ -1424,7 +1424,7 @@ calc_file_checksum(pgFile *file)
 {
 	Assert(S_ISREG(file->mode));
 
-	file->crc = pgFileGetCRC(file->path, false, false, &file->read_size);
+	file->crc = pgFileGetCRC(file->path, true, false, &file->read_size);
 	file->write_size = file->read_size;
 }
 
@@ -1547,14 +1547,14 @@ validate_one_page(Page page, pgFile *file,
 
 /* Valiate pages of datafile in backup one by one */
 bool
-check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
-				 uint32 checksum_version, uint32 backup_version)
+check_file_pages(pgFile *file, XLogRecPtr stop_lsn, uint32 checksum_version,
+				 uint32 backup_version)
 {
 	size_t		read_len = 0;
 	bool		is_valid = true;
 	FILE		*in;
 	pg_crc32	crc;
-	bool use_crc32c = (backup_version <= 20021);
+	bool		use_crc32c = backup_version <= 20021 || backup_version >= 20025;
 
 	elog(VERBOSE, "validate relation blocks for file %s", file->name);
 

Diff for: src/merge.c

@@ -524,9 +524,11 @@ merge_files(void *arg)
 				 * do that.
 				 */
 				file->write_size = pgFileSize(to_path_tmp);
-				file->crc = pgFileGetCRC(to_path_tmp, false, true, NULL);
+				file->crc = pgFileGetCRC(to_path_tmp, true, true, NULL);
 			}
 		}
+		else if (strcmp(file->name, "pg_control") == 0)
+			copy_pgcontrol_file(argument->from_root, argument->to_root, file);
 		else
 			copy_file(argument->from_root, argument->to_root, file);
 

Diff for: src/pg_probackup.c

@@ -19,7 +19,7 @@
 #include "utils/thread.h"
 #include <time.h>
 
-const char *PROGRAM_VERSION	= "2.0.24";
+const char *PROGRAM_VERSION	= "2.0.25";
 const char *PROGRAM_URL		= "https://github.com/postgrespro/pg_probackup";
 const char *PROGRAM_EMAIL	= "https://github.com/postgrespro/pg_probackup/issues";
 

Diff for: src/pg_probackup.h

@@ -96,6 +96,7 @@ do { \
 		FIN_TRADITIONAL_CRC32(crc); \
 } while (0)
 
+
 /* Information about single file (or dir) in backup */
 typedef struct pgFile
 {
@@ -539,8 +540,7 @@ extern void get_wal_file(const char *from_path, const char *to_path);
 
 extern void calc_file_checksum(pgFile *file);
 
-extern bool check_file_pages(pgFile* file,
-							 XLogRecPtr stop_lsn,
+extern bool check_file_pages(pgFile *file, XLogRecPtr stop_lsn,
 							 uint32 checksum_version, uint32 backup_version);
 /* parsexlog.c */
 extern void extractPageMap(const char *archivedir,
@@ -567,11 +567,15 @@ extern XLogRecPtr get_last_wal_lsn(const char *archivedir, XLogRecPtr start_lsn,
 /* in util.c */
 extern TimeLineID get_current_timeline(bool safe);
 extern XLogRecPtr get_checkpoint_location(PGconn *conn);
-extern uint64 get_system_identifier(char *pgdata);
+extern uint64 get_system_identifier(const char *pgdata_path);
 extern uint64 get_remote_system_identifier(PGconn *conn);
 extern uint32 get_data_checksum_version(bool safe);
+extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
 extern uint32 get_xlog_seg_size(char *pgdata_path);
-extern void set_min_recovery_point(pgFile *file, const char *backup_path, XLogRecPtr stop_backup_lsn);
+extern void set_min_recovery_point(pgFile *file, const char *backup_path,
+								   XLogRecPtr stop_backup_lsn);
+extern void copy_pgcontrol_file(const char *from_root, const char *to_root,
+								pgFile *file);
 
 extern void sanityChecks(void);
 extern void time2iso(char *buf, size_t len, time_t time);

Diff for: src/restore.c

@@ -632,6 +632,8 @@ restore_files(void *arg)
 							  false,
 							  parse_program_version(arguments->backup->program_version));
 		}
+		else if (strcmp(file->name, "pg_control") == 0)
+			copy_pgcontrol_file(from_root, instance_config.pgdata, file);
 		else
 			copy_file(from_root, instance_config.pgdata, file);
 

Diff for: src/show.c

@@ -363,7 +363,7 @@ show_instance_plain(parray *backup_list, bool show_name)
 			time2iso(row->recovery_time, lengthof(row->recovery_time),
 					 backup->recovery_time);
 		else
-			StrNCpy(row->recovery_time, "----", 4);
+			StrNCpy(row->recovery_time, "----", sizeof(row->recovery_time));
 		widths[cur] = Max(widths[cur], strlen(row->recovery_time));
 		cur++;
 
@@ -388,7 +388,7 @@ show_instance_plain(parray *backup_list, bool show_name)
 			snprintf(row->duration, lengthof(row->duration), "%.*lfs", 0,
 					 difftime(backup->end_time, backup->start_time));
 		else
-			StrNCpy(row->duration, "----", 4);
+			StrNCpy(row->duration, "----", sizeof(row->duration));
 		widths[cur] = Max(widths[cur], strlen(row->duration));
 		cur++;
 

Diff for: src/util.c

@@ -207,7 +207,7 @@ get_checkpoint_location(PGconn *conn)
 }
 
 uint64
-get_system_identifier(char *pgdata_path)
+get_system_identifier(const char *pgdata_path)
 {
 	ControlFileData ControlFile;
 	char	   *buffer;
@@ -295,7 +295,27 @@ get_data_checksum_version(bool safe)
 	return ControlFile.data_checksum_version;
 }
 
-/* MinRecoveryPoint 'as-is' is not to be trusted */
+pg_crc32c
+get_pgcontrol_checksum(const char *pgdata_path)
+{
+	ControlFileData ControlFile;
+	char	   *buffer;
+	size_t		size;
+
+	/* First fetch file... */
+	buffer = slurpFile(pgdata_path, "global/pg_control", &size, false);
+	if (buffer == NULL)
+		return 0;
+	digestControlFile(&ControlFile, buffer, size);
+	pg_free(buffer);
+
+	return ControlFile.crc;
+}
+
+/*
+ * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint
+ * 'as-is' is not to be trusted.
+ */
 void
 set_min_recovery_point(pgFile *file, const char *backup_path,
 					   XLogRecPtr stop_backup_lsn)
@@ -324,20 +344,41 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
 
 	/* Update checksum in pg_control header */
 	INIT_CRC32C(ControlFile.crc);
-	COMP_CRC32C(ControlFile.crc,
-				(char *) &ControlFile,
+	COMP_CRC32C(ControlFile.crc, (char *) &ControlFile,
 				offsetof(ControlFileData, crc));
 	FIN_CRC32C(ControlFile.crc);
 
-	/* paranoia */
-	checkControlFile(&ControlFile);
-
 	/* overwrite pg_control */
 	snprintf(fullpath, sizeof(fullpath), "%s/%s", backup_path, XLOG_CONTROL_FILE);
 	writeControlFile(&ControlFile, fullpath);
 
 	/* Update pg_control checksum in backup_list */
-	file->crc = pgFileGetCRC(fullpath, false, true, NULL);
+	file->crc = ControlFile.crc;
+
+	pg_free(buffer);
+}
+
+/*
+ * Copy pg_control file to backup. We do not apply compression to this file.
+ */
+void
+copy_pgcontrol_file(const char *from_root, const char *to_root, pgFile *file)
+{
+	ControlFileData ControlFile;
+	char	   *buffer;
+	size_t		size;
+	char		to_path[MAXPGPATH];
+
+	buffer = slurpFile(from_root, XLOG_CONTROL_FILE, &size, false);
+
+	digestControlFile(&ControlFile, buffer, size);
+
+	file->crc = ControlFile.crc;
+	file->read_size = size;
+	file->write_size = size;
+
+	join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
+	writeControlFile(&ControlFile, to_path);
 
 	pg_free(buffer);
 }

Diff for: src/validate.c

@@ -22,6 +22,7 @@ static bool corrupted_backup_found = false;
 
 typedef struct
 {
+	const char *base_path;
 	parray	   *files;
 	bool		corrupted;
 	XLogRecPtr	stop_lsn;
@@ -101,6 +102,7 @@ pgBackupValidate(pgBackup *backup)
 	{
 		validate_files_arg *arg = &(threads_args[i]);
 
+		arg->base_path = base_path;
 		arg->files = files;
 		arg->corrupted = false;
 		arg->stop_lsn = backup->stop_lsn;
@@ -223,9 +225,17 @@ pgBackupValidateFiles(void *arg)
 			 * CRC-32C algorithm.
 			 * To avoid this problem we need to use different algorithm, CRC-32 in
 			 * this case.
+			 *
+			 * Starting from 2.0.25 we calculate crc of pg_control differently.
 			 */
-			crc = pgFileGetCRC(file->path, arguments->backup_version <= 20021,
-							   true, NULL);
+			if (arguments->backup_version >= 20025 &&
+				strcmp(file->name, "pg_control") == 0)
+				crc = get_pgcontrol_checksum(arguments->base_path);
+			else
+				crc = pgFileGetCRC(file->path,
+								   arguments->backup_version <= 20021 ||
+								   arguments->backup_version >= 20025,
+								   true, NULL);
 			if (crc != file->crc)
 			{
 				elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",