-
Notifications
You must be signed in to change notification settings - Fork 87
/
Copy pathpg_probackup.h
1300 lines (1105 loc) · 46.4 KB
/
pg_probackup.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*-------------------------------------------------------------------------
*
* pg_probackup.h: Backup/Recovery manager for PostgreSQL.
*
* Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
* Portions Copyright (c) 2015-2018, Postgres Professional
*
*-------------------------------------------------------------------------
*/
#ifndef PG_PROBACKUP_H
#define PG_PROBACKUP_H
#include "postgres_fe.h"
#include "libpq-fe.h"
#include "libpq-int.h"
#include "access/xlog_internal.h"
#include "utils/pg_crc.h"
#if PG_VERSION_NUM >= 120000
#include "common/logging.h"
#endif
#ifdef FRONTEND
#undef FRONTEND
#include <port/atomics.h>
#define FRONTEND
#else
#include <port/atomics.h>
#endif
#include "utils/configuration.h"
#include "utils/logger.h"
#include "utils/remote.h"
#include "utils/parray.h"
#include "utils/pgut.h"
#include "utils/file.h"
#include "datapagemap.h"
#include "utils/thread.h"
#include "pg_probackup_state.h"
#ifdef WIN32
#define __thread __declspec(thread)
#else
#include <pthread.h>
#endif
/* Wrap the code that we're going to delete after refactoring in this define*/
#define REFACTORE_ME
/* pgut client variables and full path */
extern const char *PROGRAM_NAME;
extern const char *PROGRAM_NAME_FULL;
extern const char *PROGRAM_FULL_PATH;
extern const char *PROGRAM_URL;
extern const char *PROGRAM_EMAIL;
/* Directory/File names */
#define DATABASE_DIR "database"
#define BACKUPS_DIR "backups"
#define WAL_SUBDIR "wal"
#if PG_VERSION_NUM >= 100000
#define PG_XLOG_DIR "pg_wal"
#define PG_LOG_DIR "log"
#else
#define PG_XLOG_DIR "pg_xlog"
#define PG_LOG_DIR "pg_log"
#endif
#define PG_TBLSPC_DIR "pg_tblspc"
#define PG_GLOBAL_DIR "global"
#define BACKUP_CONTROL_FILE "backup.control"
#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf"
#define BACKUP_LOCK_FILE "backup.pid"
#define BACKUP_RO_LOCK_FILE "backup_ro.pid"
#define DATABASE_FILE_LIST "backup_content.control"
#define PG_BACKUP_LABEL_FILE "backup_label"
#define PG_TABLESPACE_MAP_FILE "tablespace_map"
#define RELMAPPER_FILENAME "pg_filenode.map"
#define EXTERNAL_DIR "external_directories/externaldir"
#define DATABASE_MAP "database_map"
#define HEADER_MAP "page_header_map"
#define HEADER_MAP_TMP "page_header_map_tmp"
#define XLOG_CONTROL_BAK_FILE XLOG_CONTROL_FILE".pbk.bak"
/* Timeout defaults */
#define ARCHIVE_TIMEOUT_DEFAULT 300
#define REPLICA_TIMEOUT_DEFAULT 300
#define LOCK_TIMEOUT 60
#define LOCK_STALE_TIMEOUT 30
#define LOG_FREQ 10
/* Directory/File permission */
#define DIR_PERMISSION (0700)
#define FILE_PERMISSION (0600)
/* 64-bit xid support for PGPRO_EE */
#ifndef PGPRO_EE
#define XID_FMT "%u"
#endif
#ifndef STDIN_FILENO
#define STDIN_FILENO 0
#define STDOUT_FILENO 1
#endif
/* stdio buffer size */
#define STDIO_BUFSIZE 65536
#define ERRMSG_MAX_LEN 2048
#define CHUNK_SIZE (128 * 1024)
#define LARGE_CHUNK_SIZE (4 * 1024 * 1024)
#define OUT_BUF_SIZE (512 * 1024)
/* retry attempts */
#define PAGE_READ_ATTEMPTS 300
/* max size of note, that can be added to backup */
#define MAX_NOTE_SIZE 1024
/* Check if an XLogRecPtr value is pointed to 0 offset */
#define XRecOffIsNull(xlrp) \
((xlrp) % XLOG_BLCKSZ == 0)
/* Text Coloring macro */
#define TC_LEN 11
#define TC_RED "\033[0;31m"
#define TC_RED_BOLD "\033[1;31m"
#define TC_BLUE "\033[0;34m"
#define TC_BLUE_BOLD "\033[1;34m"
#define TC_GREEN "\033[0;32m"
#define TC_GREEN_BOLD "\033[1;32m"
#define TC_YELLOW "\033[0;33m"
#define TC_YELLOW_BOLD "\033[1;33m"
#define TC_MAGENTA "\033[0;35m"
#define TC_MAGENTA_BOLD "\033[1;35m"
#define TC_CYAN "\033[0;36m"
#define TC_CYAN_BOLD "\033[1;36m"
#define TC_RESET "\033[0m"
typedef struct RedoParams
{
TimeLineID tli;
XLogRecPtr lsn;
uint32 checksum_version;
} RedoParams;
typedef struct PageState
{
uint16 checksum;
XLogRecPtr lsn;
} PageState;
typedef struct db_map_entry
{
Oid dbOid;
char *datname;
} db_map_entry;
/* State of pgdata in the context of its compatibility for incremental restore */
typedef enum DestDirIncrCompatibility
{
POSTMASTER_IS_RUNNING,
SYSTEM_ID_MISMATCH,
BACKUP_LABEL_EXISTS,
DEST_IS_NOT_OK,
DEST_OK
} DestDirIncrCompatibility;
typedef enum IncrRestoreMode
{
INCR_NONE,
INCR_CHECKSUM,
INCR_LSN
} IncrRestoreMode;
typedef enum PartialRestoreType
{
NONE,
INCLUDE,
EXCLUDE,
} PartialRestoreType;
typedef enum RecoverySettingsMode
{
DEFAULT, /* not set */
DONTWRITE, /* explicitly forbid to update recovery settings */
//TODO Should we always clean/preserve old recovery settings,
// or make it configurable?
PITR_REQUESTED, /* can be set based on other parameters
* if not explicitly forbidden */
} RecoverySettingsMode;
typedef enum CompressAlg
{
NOT_DEFINED_COMPRESS = 0,
NONE_COMPRESS,
PGLZ_COMPRESS,
ZLIB_COMPRESS,
} CompressAlg;
typedef enum ForkName
{
vm,
fsm,
cfm,
init,
ptrack
} ForkName;
#define INIT_FILE_CRC32(use_crc32c, crc) \
do { \
if (use_crc32c) \
INIT_CRC32C(crc); \
else \
INIT_TRADITIONAL_CRC32(crc); \
} while (0)
#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \
do { \
if (use_crc32c) \
COMP_CRC32C((crc), (data), (len)); \
else \
COMP_TRADITIONAL_CRC32(crc, data, len); \
} while (0)
#define FIN_FILE_CRC32(use_crc32c, crc) \
do { \
if (use_crc32c) \
FIN_CRC32C(crc); \
else \
FIN_TRADITIONAL_CRC32(crc); \
} while (0)
#define pg_off_t unsigned long long
/* Information about single file (or dir) in backup */
typedef struct pgFile
{
char *name; /* file or directory name */
mode_t mode; /* protection (file type and permission) */
size_t size; /* size of the file */
time_t mtime; /* file st_mtime attribute, can be used only
during backup */
size_t read_size; /* size of the portion read (if only some pages are
backed up, it's different from size) */
int64 write_size; /* size of the backed-up file. BYTES_INVALID means
that the file existed but was not backed up
because not modified since last backup. */
size_t uncompressed_size; /* size of the backed-up file before compression
* and adding block headers.
*/
/* we need int64 here to store '-1' value */
pg_crc32 crc; /* CRC value of the file, regular file only */
char *rel_path; /* relative path of the file */
char *linked; /* path of the linked file */
bool is_datafile; /* true if the file is PostgreSQL data file */
Oid tblspcOid; /* tblspcOid extracted from path, if applicable */
Oid dbOid; /* dbOid extracted from path, if applicable */
Oid relOid; /* relOid extracted from path, if applicable */
ForkName forkName; /* forkName extracted from path, if applicable */
int segno; /* Segment number for ptrack */
int n_blocks; /* number of blocks in the data file in data directory */
bool is_cfs; /* Flag to distinguish files compressed by CFS*/
bool is_database; /* Flag used strictly by ptrack 1.x backup */
int external_dir_num; /* Number of external directory. 0 if not external */
bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */
CompressAlg compress_alg; /* compression algorithm applied to the file */
volatile pg_atomic_flag lock;/* lock for synchronization of parallel threads */
datapagemap_t pagemap; /* bitmap of pages updated since previous backup
may take up to 16kB per file */
bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap,
* i.e. datafiles without _ptrack */
/* Coordinates in header map */
int n_headers; /* number of blocks in the data file in backup */
pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */
pg_off_t hdr_off; /* offset in header map */
int hdr_size; /* length of headers */
} pgFile;
typedef struct page_map_entry
{
const char *path; /* file or directory name */
char *pagemap;
size_t pagemapsize;
} page_map_entry;
/* Special values of datapagemap_t bitmapsize */
#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */
/* Return codes for check_tablespace_mapping */
#define NoTblspc 0
#define EmptyTblspc 1
#define NotEmptyTblspc 2
/* Current state of backup */
typedef enum BackupStatus
{
BACKUP_STATUS_INVALID, /* the pgBackup is invalid */
BACKUP_STATUS_OK, /* completed backup */
BACKUP_STATUS_ERROR, /* aborted because of unexpected error */
BACKUP_STATUS_RUNNING, /* running backup */
BACKUP_STATUS_MERGING, /* merging backups */
BACKUP_STATUS_MERGED, /* backup has been successfully merged and now awaits
* the assignment of new start_time */
BACKUP_STATUS_DELETING, /* data files are being deleted */
BACKUP_STATUS_DELETED, /* data files have been deleted */
BACKUP_STATUS_DONE, /* completed but not validated yet */
BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */
BACKUP_STATUS_CORRUPT /* files are corrupted, not available */
} BackupStatus;
typedef enum BackupMode
{
BACKUP_MODE_INVALID = 0,
BACKUP_MODE_DIFF_PAGE, /* incremental page backup */
BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */
BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */
BACKUP_MODE_FULL /* full backup */
} BackupMode;
typedef enum ShowFormat
{
SHOW_PLAIN,
SHOW_JSON
} ShowFormat;
/* special values of pgBackup fields */
#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.5.0"
/* update when remote agent API or behaviour changes */
#define AGENT_PROTOCOL_VERSION 20500
#define AGENT_PROTOCOL_VERSION_STR "2.5.0"
/* update only when changing storage format */
#define STORAGE_FORMAT_VERSION "2.5.0"
typedef struct ConnectionOptions
{
const char *pgdatabase;
const char *pghost;
const char *pgport;
const char *pguser;
} ConnectionOptions;
typedef struct ConnectionArgs
{
PGconn *conn;
PGcancel *cancel_conn;
} ConnectionArgs;
/* Store values for --remote-* option for 'restore_command' constructor */
typedef struct ArchiveOptions
{
const char *host;
const char *port;
const char *user;
} ArchiveOptions;
/*
* An instance configuration. It can be stored in a configuration file or passed
* from command line.
*/
typedef struct InstanceConfig
{
uint64 system_identifier;
uint32 xlog_seg_size;
char *pgdata;
char *external_dir_str;
ConnectionOptions conn_opt;
ConnectionOptions master_conn_opt;
uint32 replica_timeout; //Deprecated. Not used anywhere
/* Wait timeout for WAL segment archiving */
uint32 archive_timeout;
/* cmdline to be used as restore_command */
char *restore_command;
/* Logger parameters */
LoggerConfig logger;
/* Remote access parameters */
RemoteConfig remote;
/* Retention options. 0 disables the option. */
uint32 retention_redundancy;
uint32 retention_window;
uint32 wal_depth;
CompressAlg compress_alg;
int compress_level;
/* Archive description */
ArchiveOptions archive;
} InstanceConfig;
extern ConfigOption instance_options[];
extern InstanceConfig instance_config;
extern time_t current_time;
typedef struct PGNodeInfo
{
uint32 block_size;
uint32 wal_block_size;
uint32 checksum_version;
bool is_superuser;
bool pgpro_support;
int server_version;
char server_version_str[100];
int ptrack_version_num;
bool is_ptrack_enable;
const char *ptrack_schema; /* used only for ptrack 2.x */
} PGNodeInfo;
/* structure used for access to block header map */
typedef struct HeaderMap
{
char path[MAXPGPATH];
char path_tmp[MAXPGPATH]; /* used only in merge */
FILE *fp; /* used only for writing */
char *buf; /* buffer */
pg_off_t offset; /* current position in fp */
pthread_mutex_t mutex;
} HeaderMap;
typedef struct pgBackup pgBackup;
/* Information about single backup stored in backup.conf */
struct pgBackup
{
BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/
time_t backup_id; /* Identifier of the backup.
* Currently it's the same as start_time */
BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/
TimeLineID tli; /* timeline of start and stop backup lsns */
XLogRecPtr start_lsn; /* backup's starting transaction log location */
XLogRecPtr stop_lsn; /* backup's finishing transaction log location */
time_t start_time; /* UTC time of backup creation */
time_t merge_dest_backup; /* start_time of incremental backup with
* which this backup is merging with.
* Only available for FULL backups
* with MERGING or MERGED statuses */
time_t merge_time; /* the moment when merge was started or 0 */
time_t end_time; /* the moment when backup was finished, or the moment
* when we realized that backup is broken */
time_t recovery_time; /* Earliest moment for which you can restore
* the state of the database cluster using
* this backup */
time_t expire_time; /* Backup expiration date */
TransactionId recovery_xid; /* Earliest xid for which you can restore
* the state of the database cluster using
* this backup */
/*
* Amount of raw data. For a full backup, this is the total amount of
* data while for a differential backup this is just the difference
* of data taken.
* BYTES_INVALID means nothing was backed up.
*/
int64 data_bytes;
/* Size of WAL files needed to replay on top of this
* backup to reach the consistency.
*/
int64 wal_bytes;
/* Size of data files before applying compression and block header,
* WAL files are not included.
*/
int64 uncompressed_bytes;
/* Size of data files in PGDATA at the moment of backup. */
int64 pgdata_bytes;
CompressAlg compress_alg;
int compress_level;
/* Fields needed for compatibility check */
uint32 block_size;
uint32 wal_block_size;
uint32 checksum_version;
char program_version[100];
char server_version[100];
bool stream; /* Was this backup taken in stream mode?
* i.e. does it include all needed WAL files? */
bool from_replica; /* Was this backup taken from replica */
time_t parent_backup; /* Identifier of the previous backup.
* Which is basic backup for this
* incremental backup. */
pgBackup *parent_backup_link;
char *primary_conninfo; /* Connection parameters of the backup
* in the format suitable for recovery.conf */
char *external_dir_str; /* List of external directories,
* separated by ':' */
char *root_dir; /* Full path for root backup directory:
backup_path/instance_name/backup_id */
char *database_dir; /* Full path to directory with data files:
backup_path/instance_name/backup_id/database */
parray *files; /* list of files belonging to this backup
* must be populated explicitly */
char *note;
pg_crc32 content_crc;
/* map used for access to page headers */
HeaderMap hdr_map;
};
/* Recovery target for restore and validate subcommands */
typedef struct pgRecoveryTarget
{
time_t target_time;
/* add one more field in order to avoid deparsing target_time back */
const char *time_string;
TransactionId target_xid;
/* add one more field in order to avoid deparsing target_xid back */
const char *xid_string;
XLogRecPtr target_lsn;
/* add one more field in order to avoid deparsing target_lsn back */
const char *lsn_string;
TimeLineID target_tli;
bool target_inclusive;
bool inclusive_specified;
const char *target_stop;
const char *target_name;
const char *target_action;
} pgRecoveryTarget;
/* Options needed for restore and validate commands */
typedef struct pgRestoreParams
{
bool force;
bool is_restore;
bool no_validate;
bool restore_as_replica;
//TODO maybe somehow add restore_as_replica as one of RecoverySettingsModes
RecoverySettingsMode recovery_settings_mode;
bool skip_external_dirs;
bool skip_block_validation; //Start using it
const char *restore_command;
const char *primary_slot_name;
const char *primary_conninfo;
/* options for incremental restore */
IncrRestoreMode incremental_mode;
XLogRecPtr shift_lsn;
/* options for partial restore */
PartialRestoreType partial_restore_type;
parray *partial_db_list;
} pgRestoreParams;
/* Options needed for set-backup command */
typedef struct pgSetBackupParams
{
int64 ttl; /* amount of time backup must be pinned
* -1 - do nothing
* 0 - disable pinning
*/
time_t expire_time; /* Point in time until backup
* must be pinned.
*/
char *note;
} pgSetBackupParams;
typedef struct
{
PGNodeInfo *nodeInfo;
const char *from_root;
const char *to_root;
const char *external_prefix;
parray *files_list;
parray *prev_filelist;
parray *external_dirs;
XLogRecPtr prev_start_lsn;
ConnectionArgs conn_arg;
int thread_num;
HeaderMap *hdr_map;
/*
* Return value from the thread.
* 0 means there is no error, 1 - there is an error.
*/
int ret;
} backup_files_arg;
typedef struct timelineInfo timelineInfo;
/* struct to collect info about timelines in WAL archive */
struct timelineInfo {
TimeLineID tli; /* this timeline */
TimeLineID parent_tli; /* parent timeline. 0 if none */
timelineInfo *parent_link; /* link to parent timeline */
XLogRecPtr switchpoint; /* if this timeline has a parent, then
* switchpoint contains switchpoint LSN,
* otherwise 0 */
XLogSegNo begin_segno; /* first present segment in this timeline */
XLogSegNo end_segno; /* last present segment in this timeline */
size_t n_xlog_files; /* number of segments (only really existing)
* does not include lost segments */
size_t size; /* space on disk taken by regular WAL files */
parray *backups; /* array of pgBackup sturctures with info
* about backups belonging to this timeline */
parray *xlog_filelist; /* array of ordinary WAL segments, '.partial'
* and '.backup' files belonging to this timeline */
parray *lost_segments; /* array of intervals of lost segments */
parray *keep_segments; /* array of intervals of segments used by WAL retention */
pgBackup *closest_backup; /* link to valid backup, closest to timeline */
pgBackup *oldest_backup; /* link to oldest backup on timeline */
XLogRecPtr anchor_lsn; /* LSN belonging to the oldest segno to keep for 'wal-depth' */
TimeLineID anchor_tli; /* timeline of anchor_lsn */
};
typedef struct xlogInterval
{
XLogSegNo begin_segno;
XLogSegNo end_segno;
} xlogInterval;
typedef struct lsnInterval
{
TimeLineID tli;
XLogRecPtr begin_lsn;
XLogRecPtr end_lsn;
} lsnInterval;
typedef enum xlogFileType
{
SEGMENT,
TEMP_SEGMENT,
PARTIAL_SEGMENT,
BACKUP_HISTORY_FILE
} xlogFileType;
typedef struct xlogFile
{
pgFile file;
XLogSegNo segno;
xlogFileType type;
bool keep; /* Used to prevent removal of WAL segments
* required by ARCHIVE backups. */
} xlogFile;
/*
* When copying datafiles to backup we validate and compress them block
* by block. Thus special header is required for each data block.
*/
typedef struct BackupPageHeader
{
BlockNumber block; /* block number */
int32 compressed_size;
} BackupPageHeader;
/* 4MB for 1GB file */
typedef struct BackupPageHeader2
{
XLogRecPtr lsn;
int32 block; /* block number */
int32 pos; /* position in backup file */
uint16 checksum;
} BackupPageHeader2;
typedef struct StopBackupCallbackParams
{
PGconn *conn;
int server_version;
} StopBackupCallbackParams;
/* Special value for compressed_size field */
#define PageIsOk 0
#define SkipCurrentPage -1
#define PageIsTruncated -2
#define PageIsCorrupted -3 /* used by checkdb */
/*
* Return timeline, xlog ID and record offset from an LSN of the type
* 0/B000188, usual result from pg_stop_backup() and friends.
*/
#define XLogDataFromLSN(data, xlogid, xrecoff) \
sscanf(data, "%X/%X", xlogid, xrecoff)
#define IsCompressedXLogFileName(fname) \
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0)
#if PG_VERSION_NUM >= 110000
#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \
XLogSegmentOffset(xlogptr, wal_segsz_bytes)
#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \
XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \
XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \
XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes)
#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \
snprintf(fname, 20, "%08X%08X", \
(uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \
(uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes)))
#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \
logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg
#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#else
#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \
((xlogptr) & ((XLogSegSize) - 1))
#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \
XLByteToSeg(xlrp, logSegNo)
#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \
XLogSegNoOffsetToRecPtr(segno, offset, dest)
#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFileName(fname, tli, logSegNo)
#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \
XLByteInSeg(xlrp, logSegNo)
#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \
snprintf(fname, 20, "%08X%08X",\
(uint32) ((logSegNo) / XLogSegmentsPerXLogId), \
(uint32) ((logSegNo) % XLogSegmentsPerXLogId))
#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \
logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg
#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \
XLogFromFileName(fname, tli, logSegNo)
#endif
#define IsPartialCompressXLogFileName(fname) \
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0)
#define IsTempXLogFileName(fname) \
(strlen(fname) == XLOG_FNAME_LEN + strlen(".part") && \
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0)
#define IsTempCompressXLogFileName(fname) \
(strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \
strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \
strcmp((fname) + XLOG_FNAME_LEN, ".gz.part") == 0)
#define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0)
/* common options */
extern pid_t my_pid;
extern __thread int my_thread_num;
extern int num_threads;
extern bool stream_wal;
extern bool show_color;
extern bool progress;
extern bool is_archive_cmd; /* true for archive-{get,push} */
#if PG_VERSION_NUM >= 100000
/* In pre-10 'replication_slot' is defined in receivelog.h */
extern char *replication_slot;
#endif
extern bool temp_slot;
/* backup options */
extern bool smooth_checkpoint;
/* remote probackup options */
extern char* remote_agent;
extern bool exclusive_backup;
/* delete options */
extern bool delete_wal;
extern bool delete_expired;
extern bool merge_expired;
extern bool dry_run;
/* ===== instanceState ===== */
typedef struct InstanceState
{
/* catalog, this instance belongs to */
CatalogState *catalog_state;
char instance_name[MAXPGPATH]; //previously global var instance_name
/* $BACKUP_PATH/backups/instance_name */
char instance_backup_subdir_path[MAXPGPATH];
/* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */
char instance_config_path[MAXPGPATH];
/* $BACKUP_PATH/backups/instance_name */
char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path
/* TODO: Make it more specific */
PGconn *conn;
//TODO split into some more meaningdul parts
InstanceConfig *config;
} InstanceState;
/* ===== instanceState (END) ===== */
/* show options */
extern ShowFormat show_format;
/* checkdb options */
extern bool heapallindexed;
extern bool skip_block_validation;
/* current settings */
extern pgBackup current;
/* argv of the process */
extern char** commands_args;
/* in backup.c */
extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params,
bool no_validate, bool no_sync, bool backup_logs);
extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt,
char *pgdata);
extern BackupMode parse_backup_mode(const char *value);
extern const char *deparse_backup_mode(BackupMode mode);
extern void process_block_change(ForkNumber forknum, RelFileNode rnode,
BlockNumber blkno);
extern char *pg_ptrack_get_block(ConnectionArgs *arguments,
Oid dbOid, Oid tblsOid, Oid relOid,
BlockNumber blknum, size_t *result_size,
int ptrack_version_num, const char *ptrack_schema);
/* in restore.c */
extern int do_restore_or_validate(InstanceState *instanceState,
time_t target_backup_id,
pgRecoveryTarget *rt,
pgRestoreParams *params,
bool no_sync);
extern bool satisfy_timeline(const parray *timelines, const pgBackup *backup);
extern bool satisfy_recovery_target(const pgBackup *backup,
const pgRecoveryTarget *rt);
extern pgRecoveryTarget *parseRecoveryTargetOptions(
const char *target_time, const char *target_xid,
const char *target_inclusive, TimeLineID target_tli, const char* target_lsn,
const char *target_stop, const char *target_name,
const char *target_action);
extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list,
PartialRestoreType partial_restore_type);
extern parray *get_backup_filelist(pgBackup *backup, bool strict);
extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict);
extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli);
/* in merge.c */
extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync);
extern void merge_backups(pgBackup *backup, pgBackup *next_backup);
extern void merge_chain(InstanceState *instanceState, parray *parent_chain,
pgBackup *full_backup, pgBackup *dest_backup,
bool no_validate, bool no_sync);
extern parray *read_database_map(pgBackup *backup);
/* in init.c */
extern int do_init(CatalogState *catalogState);
extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance);
/* in archive.c */
extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wal_file_path,
char *wal_file_name, int batch_size, bool overwrite,
bool no_sync, bool no_ready_rename);
extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path,
char *wal_file_name, int batch_size, bool validate_wal);
/* in configure.c */
extern void do_show_config(void);
extern void do_set_config(InstanceState *instanceState, bool missing_ok);
extern void init_config(InstanceConfig *config, const char *instance_name);
extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState);
/* in show.c */
extern int do_show(CatalogState *catalogState, InstanceState *instanceState,
time_t requested_backup_id, bool show_archive);
/* in delete.c */
extern void do_delete(InstanceState *instanceState, time_t backup_id);
extern void delete_backup_files(pgBackup *backup);
extern void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync);
extern int do_delete_instance(InstanceState *instanceState);
extern void do_delete_status(InstanceState *instanceState,
InstanceConfig *instance_config, const char *status);
/* in fetch.c */
extern char *slurpFile(const char *datadir,
const char *path,
size_t *filesize,
bool safe,
fio_location location);
extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize);
/* in help.c */
extern void help_print_version(void);
extern void help_pg_probackup(void);
extern void help_command(ProbackupSubcmd const subcmd);
/* in validate.c */
extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params);
extern int do_validate_all(CatalogState *catalogState, InstanceState *instanceState);
extern int validate_one_page(Page page, BlockNumber absolute_blkno,
XLogRecPtr stop_lsn, PageState *page_st,
uint32 checksum_version);
extern bool validate_tablespace_map(pgBackup *backup, bool no_validate);
extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list);
/* return codes for validate_one_page */
/* TODO: use enum */
#define PAGE_IS_VALID (-1)
#define PAGE_IS_NOT_FOUND (-2)
#define PAGE_IS_ZEROED (-3)
#define PAGE_HEADER_IS_INVALID (-4)
#define PAGE_CHECKSUM_MISMATCH (-5)
#define PAGE_LSN_FROM_FUTURE (-6)
/* in catalog.c */
extern pgBackup *read_backup(const char *root_dir);
extern void write_backup(pgBackup *backup, bool strict);
extern void write_backup_status(pgBackup *backup, BackupStatus status,
bool strict);
extern void write_backup_data_bytes(pgBackup *backup);
extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive);
extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color);
extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode);
extern parray *catalog_get_instance_list(CatalogState *catalogState);
extern parray *catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id);
extern void catalog_lock_backup_list(parray *backup_list, int from_idx,
int to_idx, bool strict, bool exclusive);
extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
TimeLineID tli,
time_t current_start_time);
extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list,
TimeLineID current_tli, time_t current_start_time,
InstanceConfig *instance);
extern timelineInfo *timelineInfoNew(TimeLineID tli);
extern void timelineInfoFree(void *tliInfo);
extern parray *catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance);
extern void do_set_backup(InstanceState *instanceState, time_t backup_id,
pgSetBackupParams *set_backup_params);
extern void pin_backup(pgBackup *target_backup,
pgSetBackupParams *set_backup_params);
extern void add_note(pgBackup *target_backup, char *note);
extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc);
extern void write_backup_filelist(pgBackup *backup, parray *files,
const char *root, parray *external_list, bool sync);
extern void pgBackupCreateDir(pgBackup *backup, const char *backup_instance_path);
extern void pgNodeInit(PGNodeInfo *node);
extern void pgBackupInit(pgBackup *backup);
extern void pgBackupFree(void *backup);
extern int pgBackupCompareId(const void *f1, const void *f2);
extern int pgBackupCompareIdDesc(const void *f1, const void *f2);
extern int pgBackupCompareIdEqual(const void *l, const void *r);
extern pgBackup* find_parent_full_backup(pgBackup *current_backup);
extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup);
/* return codes for scan_parent_chain */
#define ChainIsBroken 0
#define ChainIsInvalid 1
#define ChainIsOk 2
extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive);
extern bool is_prolific(parray *backup_list, pgBackup *target_backup);
extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list);
extern bool launch_agent(void);
extern void launch_ssh(char* argv[]);
extern void wait_ssh(void);
#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS
#define COMPRESS_LEVEL_DEFAULT 1