tests: added tests.external.ExternalTest.test_smart_restore_externals

Author: gsmolk

Diff for: tests/external.py

@@ -38,7 +38,7 @@ def test_external_simple(self):
 
         # take FULL backup with external directory pointing to a file
         file_path = os.path.join(core_dir, 'file')
-        open(file_path,"w+")
+        open(file_path, "w+")
 
         try:
             self.backup_node(
@@ -2284,3 +2284,82 @@ def test_merge_external_dir_string_order(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
+
+    # @unittest.skip("skip")
+    def test_smart_restore_externals(self):
+        """
+        make node, create database, take full backup with externals,
+        take incremental backup without externals and restore it,
+        make sure that files from externals are not copied during restore
+        https://github.com/postgrespro/pg_probackup/issues/63
+        """
+        fname = self.id().split('.')[3]
+        node = self.make_simple_node(
+            base_dir=os.path.join(module_name, fname, 'node'),
+            set_replication=True,
+            initdb_params=['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        # fill external directories with data
+        tmp_id = self.backup_node(backup_dir, 'node', node)
+
+        external_dir_1 = self.get_tblspace_path(node, 'external_dir_1')
+        external_dir_2 = self.get_tblspace_path(node, 'external_dir_2')
+
+        self.restore_node(
+            backup_dir, 'node', node, backup_id=tmp_id,
+            data_dir=external_dir_1, options=["-j", "4"])
+
+        self.restore_node(
+            backup_dir, 'node', node, backup_id=tmp_id,
+            data_dir=external_dir_2, options=["-j", "4"])
+
+        self.delete_pb(backup_dir, 'node', backup_id=tmp_id)
+
+        # create database
+        node.safe_psql(
+            "postgres",
+            "CREATE DATABASE testdb")
+
+        # take FULL backup
+        full_id = self.backup_node(backup_dir, 'node', node)
+
+        # drop database
+        node.safe_psql(
+            "postgres",
+            "DROP DATABASE testdb")
+
+        # take PAGE backup
+        page_id = self.backup_node(
+            backup_dir, 'node', node, backup_type='page')
+
+        # restore PAGE backup
+        node.cleanup()
+        self.restore_node(
+            backup_dir, 'node', node, backup_id=page_id,
+            options=['--no-validate', '--log-level-file=VERBOSE'])
+
+        logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log')
+        with open(logfile, 'r') as f:
+                logfile_content = f.read()
+
+        # get delta between FULL and PAGE filelists
+        filelist_full = self.get_backup_filelist(
+            backup_dir, 'node', full_id)
+
+        filelist_page = self.get_backup_filelist(
+            backup_dir, 'node', page_id)
+
+        filelist_diff = self.get_backup_filelist_diff(
+            filelist_full, filelist_page)
+
+        for file in filelist_diff:
+            self.assertNotIn(file, logfile_content)
+
+        # Clean after yourself
+        self.del_test_dir(module_name, fname)

Diff for: tests/restore.py

@@ -2168,6 +2168,3 @@ def test_smart_restore(self):
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)
-
-
-# smart restore of external dirs

Diff for: tests/retention.py

@@ -67,7 +67,7 @@ def test_retention_redundancy_1(self):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    #@unittest.skip("skip")
+    # @unittest.skip("skip")
     def test_retention_window_2(self):
         """purge backups using window-based retention policy"""
         fname = self.id().split('.')[3]
@@ -120,7 +120,7 @@ def test_retention_window_2(self):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-    #@unittest.skip("skip")
+    # @unittest.skip("skip")
     def test_retention_window_3(self):
         """purge all backups using window-based retention policy"""
         fname = self.id().split('.')[3]
@@ -134,8 +134,7 @@ def test_retention_window_3(self):
         self.set_archiving(backup_dir, 'node', node)
         node.slow_start()
 
-
-        # Take FULL BACKUP
+        # take FULL BACKUP
         backup_id_1 = self.backup_node(backup_dir, 'node', node)
 
         # Take second FULL BACKUP
@@ -144,7 +143,6 @@ def test_retention_window_3(self):
         # Take third FULL BACKUP
         backup_id_3 = self.backup_node(backup_dir, 'node', node)
 
-
         backups = os.path.join(backup_dir, 'backups', 'node')
         for backup in os.listdir(backups):
             if backup == 'pg_probackup.conf':
@@ -183,8 +181,7 @@ def test_retention_window_4(self):
         self.set_archiving(backup_dir, 'node', node)
         node.slow_start()
 
-
-        # Take FULL BACKUPs
+        # take FULL BACKUPs
         backup_id_1 = self.backup_node(backup_dir, 'node', node)
 
         backup_id_2 = self.backup_node(backup_dir, 'node', node)
@@ -245,9 +242,7 @@ def test_window_expire_interleaved_incremental_chains(self):
         self.set_archiving(backup_dir, 'node', node)
         node.slow_start()
 
-
-        # Take FULL BACKUPs
-
+        # take FULL BACKUPs
         backup_id_a = self.backup_node(backup_dir, 'node', node)
         backup_id_b = self.backup_node(backup_dir, 'node', node)
 
@@ -334,7 +329,6 @@ def test_window_expire_interleaved_incremental_chains(self):
                 conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format(
                     datetime.now() - timedelta(days=3)))
 
-
         self.delete_expired(
             backup_dir, 'node',
             options=['--retention-window=1', '--expired'])
@@ -361,8 +355,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self):
         self.set_archiving(backup_dir, 'node', node)
         node.slow_start()
 
-
-        # Take FULL BACKUPs
+        # take FULL BACKUPs
         backup_id_a = self.backup_node(backup_dir, 'node', node)
         backup_id_b = self.backup_node(backup_dir, 'node', node)
 
@@ -710,7 +703,6 @@ def test_window_merge_interleaved_incremental_chains_1(self):
             backup_dir, 'node',
             options=['--retention-window=1', '--expired', '--merge-expired'])
 
-
         self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2)
 
         self.assertEqual(
@@ -1004,7 +996,6 @@ def test_window_merge_multiple_descendants(self):
             "INFO: Delete: {0}".format(
                 page_id_a1), output)
 
-
         self.assertEqual(
             self.show_pb(backup_dir, 'node')[2]['id'],
             page_id_b3)
@@ -1119,7 +1110,6 @@ def test_window_chains(self):
         # Clean after yourself
         self.del_test_dir(module_name, fname)
 
-
     # @unittest.skip("skip")
     def test_window_chains_1(self):
         """
@@ -1245,4 +1235,4 @@ def test_window_error_backups(self):
             backup_dir, 'node', node, backup_type='page')
 
         # Change FULLb backup status to ERROR
-        #self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')
+        # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR')