Skip to content

Commit 23cf692

Browse files
za-arthurza-arthur
committed
PGPRO-2096: Use CRC-32 instead of CRC-32C
Using CRC-32C to calculate checksum of pg_control gives same value for different backups. It might be because pg_control stores its content plus checksum of the content.
1 parent effc719 commit 23cf692

File tree

5 files changed

+45
-35
lines changed

5 files changed

+45
-35
lines changed

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
/tests/helpers/*pyc
3434

3535
# Extra files
36+
/src/pg_crc.c
3637
/src/datapagemap.c
3738
/src/datapagemap.h
3839
/src/logging.h

Diff for: ‎Makefile

+21-12
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,23 @@
11
PROGRAM = pg_probackup
2-
OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
3-
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \
4-
src/pg_probackup.o src/restore.o src/show.o \
5-
src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \
6-
src/xlogreader.o src/streamutil.o src/receivelog.o \
7-
src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o \
8-
src/utils/json.o src/utils/thread.o src/merge.o
92

10-
EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \
11-
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h
3+
# utils
4+
OBJS = src/utils/json.o src/utils/logger.o src/utils/parray.o \
5+
src/utils/pgut.o src/utils/thread.o
126

13-
INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h
7+
OBJS += src/archive.o src/backup.o src/catalog.o src/configure.o src/data.o \
8+
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o src/merge.o \
9+
src/parsexlog.o src/pg_probackup.o src/restore.o src/show.o src/util.o \
10+
src/validate.o
11+
12+
# borrowed files
13+
OBJS += src/pg_crc.o src/datapagemap.o src/receivelog.o src/streamutil.o \
14+
src/xlogreader.o
15+
16+
EXTRA_CLEAN = src/pg_crc.c src/datapagemap.c src/datapagemap.h src/logging.h \
17+
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h \
18+
src/xlogreader.c
19+
20+
INCLUDES = src/datapagemap.h src/logging.h src/streamutil.h src/receivelog.h
1421

1522
ifdef USE_PGXS
1623
PG_CONFIG = pg_config
@@ -46,14 +53,14 @@ all: checksrcdir $(INCLUDES);
4653

4754
$(PROGRAM): $(OBJS)
4855

49-
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
50-
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
5156
src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
5257
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@
5358
src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h
5459
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@
5560
src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h
5661
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@
62+
src/pg_crc.c: $(top_srcdir)/src/backend/utils/hash/pg_crc.c
63+
rm -f $@ && $(LN_S) $(srchome)/src/backend/utils/hash/pg_crc.c $@
5764
src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c
5865
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@
5966
src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h
@@ -62,6 +69,8 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
6269
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@
6370
src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
6471
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
72+
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
73+
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
6574

6675

6776
ifeq (,$(filter 9.5 9.6,$(MAJORVERSION)))

Diff for: ‎src/backup.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
306306
to_path, strerror(errno_tmp));
307307
}
308308

309-
INIT_CRC32C(file->crc);
309+
INIT_TRADITIONAL_CRC32(file->crc);
310310

311311
/* read from stream and write to backup file */
312312
while (1)
@@ -332,14 +332,14 @@ remote_copy_file(PGconn *conn, pgFile* file)
332332
{
333333
write_buffer_size = Min(row_length, sizeof(buf));
334334
memcpy(buf, copybuf, write_buffer_size);
335-
COMP_CRC32C(file->crc, buf, write_buffer_size);
335+
COMP_TRADITIONAL_CRC32(file->crc, buf, write_buffer_size);
336336

337337
/* TODO calc checksum*/
338338
if (fwrite(buf, 1, write_buffer_size, out) != write_buffer_size)
339339
{
340340
errno_tmp = errno;
341341
/* oops */
342-
FIN_CRC32C(file->crc);
342+
FIN_TRADITIONAL_CRC32(file->crc);
343343
fclose(out);
344344
PQfinish(conn);
345345
elog(ERROR, "cannot write to \"%s\": %s", to_path,
@@ -363,7 +363,7 @@ remote_copy_file(PGconn *conn, pgFile* file)
363363
}
364364

365365
file->write_size = (int64) file->read_size;
366-
FIN_CRC32C(file->crc);
366+
FIN_TRADITIONAL_CRC32(file->crc);
367367

368368
fclose(out);
369369
}
@@ -2137,7 +2137,7 @@ backup_files(void *arg)
21372137
continue;
21382138
}
21392139
}
2140-
else
2140+
else
21412141
{
21422142
bool skip = false;
21432143

@@ -2147,7 +2147,7 @@ backup_files(void *arg)
21472147
{
21482148
calc_file_checksum(file);
21492149
/* ...and checksum is the same... */
2150-
if (EQ_CRC32C(file->crc, (*prev_file)->crc))
2150+
if (EQ_TRADITIONAL_CRC32(file->crc, (*prev_file)->crc))
21512151
skip = true; /* ...skip copying file. */
21522152
}
21532153
if (skip ||

Diff for: ‎src/data.c

+13-13
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@ compress_and_backup_page(pgFile *file, BlockNumber blknum,
416416
blknum, header.compressed_size, write_buffer_size); */
417417

418418
/* Update CRC */
419-
COMP_CRC32C(*crc, write_buffer, write_buffer_size);
419+
COMP_TRADITIONAL_CRC32(*crc, write_buffer, write_buffer_size);
420420

421421
/* write data page */
422422
if(fwrite(write_buffer, 1, write_buffer_size, out) != write_buffer_size)
@@ -476,13 +476,13 @@ backup_data_file(backup_files_arg* arguments,
476476
/* reset size summary */
477477
file->read_size = 0;
478478
file->write_size = 0;
479-
INIT_CRC32C(file->crc);
479+
INIT_TRADITIONAL_CRC32(file->crc);
480480

481481
/* open backup mode file for read */
482482
in = fopen(file->path, PG_BINARY_R);
483483
if (in == NULL)
484484
{
485-
FIN_CRC32C(file->crc);
485+
FIN_TRADITIONAL_CRC32(file->crc);
486486

487487
/*
488488
* If file is not found, this is not en error.
@@ -587,7 +587,7 @@ backup_data_file(backup_files_arg* arguments,
587587
to_path, strerror(errno));
588588
fclose(in);
589589

590-
FIN_CRC32C(file->crc);
590+
FIN_TRADITIONAL_CRC32(file->crc);
591591

592592
/*
593593
* If we have pagemap then file in the backup can't be a zero size.
@@ -839,7 +839,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
839839
struct stat st;
840840
pg_crc32 crc;
841841

842-
INIT_CRC32C(crc);
842+
INIT_TRADITIONAL_CRC32(crc);
843843

844844
/* reset size summary */
845845
file->read_size = 0;
@@ -849,7 +849,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
849849
in = fopen(file->path, PG_BINARY_R);
850850
if (in == NULL)
851851
{
852-
FIN_CRC32C(crc);
852+
FIN_TRADITIONAL_CRC32(crc);
853853
file->crc = crc;
854854

855855
/* maybe deleted, it's not error */
@@ -898,7 +898,7 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
898898
strerror(errno_tmp));
899899
}
900900
/* update CRC */
901-
COMP_CRC32C(crc, buf, read_len);
901+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
902902

903903
file->read_size += read_len;
904904
}
@@ -925,14 +925,14 @@ copy_file(const char *from_root, const char *to_root, pgFile *file)
925925
strerror(errno_tmp));
926926
}
927927
/* update CRC */
928-
COMP_CRC32C(crc, buf, read_len);
928+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
929929

930930
file->read_size += read_len;
931931
}
932932

933933
file->write_size = (int64) file->read_size;
934934
/* finish CRC calculation and store into pgFile */
935-
FIN_CRC32C(crc);
935+
FIN_TRADITIONAL_CRC32(crc);
936936
file->crc = crc;
937937

938938
/* update file permission */
@@ -1350,7 +1350,7 @@ calc_file_checksum(pgFile *file)
13501350
pg_crc32 crc;
13511351

13521352
Assert(S_ISREG(file->mode));
1353-
INIT_CRC32C(crc);
1353+
INIT_TRADITIONAL_CRC32(crc);
13541354

13551355
/* reset size summary */
13561356
file->read_size = 0;
@@ -1360,7 +1360,7 @@ calc_file_checksum(pgFile *file)
13601360
in = fopen(file->path, PG_BINARY_R);
13611361
if (in == NULL)
13621362
{
1363-
FIN_CRC32C(crc);
1363+
FIN_TRADITIONAL_CRC32(crc);
13641364
file->crc = crc;
13651365

13661366
/* maybe deleted, it's not error */
@@ -1387,7 +1387,7 @@ calc_file_checksum(pgFile *file)
13871387
break;
13881388

13891389
/* update CRC */
1390-
COMP_CRC32C(crc, buf, read_len);
1390+
COMP_TRADITIONAL_CRC32(crc, buf, read_len);
13911391

13921392
file->write_size += read_len;
13931393
file->read_size += read_len;
@@ -1402,7 +1402,7 @@ calc_file_checksum(pgFile *file)
14021402
}
14031403

14041404
/* finish CRC calculation and store into pgFile */
1405-
FIN_CRC32C(crc);
1405+
FIN_TRADITIONAL_CRC32(crc);
14061406
file->crc = crc;
14071407

14081408
fclose(in);

Diff for: ‎src/dir.c

+4-4
Original file line numberDiff line numberDiff line change
@@ -274,20 +274,20 @@ pgFileGetCRC(const char *file_path)
274274
file_path, strerror(errno));
275275

276276
/* calc CRC of backup file */
277-
INIT_CRC32C(crc);
277+
INIT_TRADITIONAL_CRC32(crc);
278278
while ((len = fread(buf, 1, sizeof(buf), fp)) == sizeof(buf))
279279
{
280280
if (interrupted)
281281
elog(ERROR, "interrupted during CRC calculation");
282-
COMP_CRC32C(crc, buf, len);
282+
COMP_TRADITIONAL_CRC32(crc, buf, len);
283283
}
284284
errno_tmp = errno;
285285
if (!feof(fp))
286286
elog(WARNING, "cannot read \"%s\": %s", file_path,
287287
strerror(errno_tmp));
288288
if (len > 0)
289-
COMP_CRC32C(crc, buf, len);
290-
FIN_CRC32C(crc);
289+
COMP_TRADITIONAL_CRC32(crc, buf, len);
290+
FIN_TRADITIONAL_CRC32(crc);
291291

292292
fclose(fp);
293293

0 commit comments

Comments
 (0)