tests: now ptrack tests by default are disabled, they can be enables via PG_PROBACKUP_PTRACK env variable. Multiple fixes, especially for PG 9.5

Author: gsmolk

tests/Readme.md

@@ -29,6 +29,9 @@ Remote backup depends on key authentithication to local machine via ssh as curre
 Run suit of basic simple tests:
  export PG_PROBACKUP_TEST_BASIC=ON
 
+Run ptrack tests:
+ export PG_PROBACKUP_PTRACK=ON
+
 
 Usage:
  pip install testgres

tests/__init__.py

@@ -37,7 +37,6 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(merge))
     suite.addTests(loader.loadTestsFromModule(option))
     suite.addTests(loader.loadTestsFromModule(page))
-#    suite.addTests(loader.loadTestsFromModule(ptrack))
     suite.addTests(loader.loadTestsFromModule(pgpro560))
     suite.addTests(loader.loadTestsFromModule(pgpro589))
     suite.addTests(loader.loadTestsFromModule(pgpro2068))
@@ -50,6 +49,10 @@ def load_tests(loader, tests, pattern):
     suite.addTests(loader.loadTestsFromModule(time_stamp))
     suite.addTests(loader.loadTestsFromModule(validate))
 
+    if 'PG_PROBACKUP_PTRACK' in os.environ:
+        if os.environ['PG_PROBACKUP_PTRACK'] == 'ON':
+            suite.addTests(loader.loadTestsFromModule(ptrack))
+
     return suite
 
 #   test_pgpro434_2 unexpected success

tests/archive.py

@@ -228,10 +228,8 @@ def test_pgpro434_3(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True,
-            initdb_params=['--data-checksums'],
-            pg_options={
-                'checkpoint_timeout': '30s'}
-            )
+            initdb_params=['--data-checksums'])
+
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
         self.set_archiving(backup_dir, 'node', node)
@@ -258,18 +256,19 @@ def test_pgpro434_3(self):
         log_file = os.path.join(backup_dir, 'log/pg_probackup.log')
         with open(log_file, 'r') as f:
             log_content = f.read()
-            self.assertNotIn(
-                "ERROR: pg_stop_backup doesn't answer",
-                log_content,
-                "pg_stop_backup timeouted")
+
+        self.assertIn(
+            "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it",
+            log_content)
 
         log_file = os.path.join(node.logs_dir, 'postgresql.log')
         with open(log_file, 'r') as f:
             log_content = f.read()
-            self.assertNotIn(
-                'FailedAssertion',
-                log_content,
-                'PostgreSQL crashed because of a failed assert')
+
+        self.assertNotIn(
+            'FailedAssertion',
+            log_content,
+            'PostgreSQL crashed because of a failed assert')
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/backup.py

@@ -15,6 +15,9 @@ class BackupTest(ProbackupTest, unittest.TestCase):
     # PGPRO-707
     def test_backup_modes_archive(self):
         """standart backup modes with ARCHIVE WAL method"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -109,6 +112,9 @@ def test_smooth_checkpoint(self):
     # @unittest.skip("skip")
     def test_incremental_backup_without_full(self):
         """page-level backup without validated full backup"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -168,8 +174,7 @@ def test_incremental_backup_corrupt_full(self):
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
-            initdb_params=['--data-checksums'],
-            pg_options={'ptrack_enable': 'on'})
+            initdb_params=['--data-checksums'])
 
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         self.init_pb(backup_dir)
@@ -230,6 +235,9 @@ def test_incremental_backup_corrupt_full(self):
     # @unittest.skip("skip")
     def test_ptrack_threads(self):
         """ptrack multi thread backup mode"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -258,6 +266,9 @@ def test_ptrack_threads(self):
     # @unittest.skip("skip")
     def test_ptrack_threads_stream(self):
         """ptrack multi thread backup mode and stream"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -287,6 +298,9 @@ def test_ptrack_threads_stream(self):
     # @unittest.skip("skip")
     def test_page_corruption_heal_via_ptrack_1(self):
         """make node, corrupt some page, check that backup failed"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -343,6 +357,9 @@ def test_page_corruption_heal_via_ptrack_1(self):
     # @unittest.skip("skip")
     def test_page_corruption_heal_via_ptrack_2(self):
         """make node, corrupt some page, check that backup failed"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
@@ -988,6 +1005,9 @@ def test_drop_rel_during_backup_page(self):
     # @unittest.skip("skip")
     def test_drop_rel_during_backup_ptrack(self):
         """"""
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
@@ -1492,10 +1512,12 @@ def test_backup_with_least_privileges_role(self):
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True,
-            # pg_options={'ptrack_enable': 'on'},
             initdb_params=['--data-checksums'],
+            pg_options={
+                'archive_timeout': '30s'})
 
-        )
+        if self.ptrack:
+            node.append_conf('postgresql.auto.conf', 'ptrack_enable = on')
 
         self.init_pb(backup_dir)
         self.add_instance(backup_dir, 'node', node)
@@ -1506,58 +1528,87 @@ def test_backup_with_least_privileges_role(self):
             'postgres',
             'CREATE DATABASE backupdb')
 
-        node.safe_psql(
-            'backupdb',
-            "REVOKE ALL ON DATABASE backupdb from PUBLIC; "
-            "REVOKE ALL ON SCHEMA public from PUBLIC; "
-            "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
-            "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
-            "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
-            "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
-            "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
-            "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
-            "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
-            "REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
-            "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
-            "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
-            "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
-            "CREATE ROLE backup WITH LOGIN REPLICATION; "
-            "GRANT CONNECT ON DATABASE backupdb to backup; "
-            "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
-            "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
-            # for partial restore, checkdb and ptrack
-            "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
-            # for exclusive backup for PG 9.5 and ptrack
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
-            "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
-        )
-
-        # ptrack functions
-        # for fname in [
-        #         'oideq(oid, oid)',
-        #         'ptrack_version()',
-        #         'pg_ptrack_clear()',
-        #         'pg_ptrack_control_lsn()',
-        #         'pg_ptrack_get_and_clear_db(oid, oid)',
-        #         'pg_ptrack_get_and_clear(oid, oid)',
-        #         'pg_ptrack_get_block_2(oid, oid, oid, bigint)']:
-        #     try:
-        #         node.safe_psql(
-        #             "backupdb",
-        #             "GRANT EXECUTE ON FUNCTION pg_catalog.{0} "
-        #             "TO backup".format(fname))
-        #     except:
-        #         pass
+        if self.get_version(node) < 90600:
+            node.safe_psql(
+                'backupdb',
+                "REVOKE ALL ON DATABASE backupdb from PUBLIC; "
+                "REVOKE ALL ON SCHEMA public from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
+                "CREATE ROLE backup WITH LOGIN REPLICATION; "
+                "GRANT CONNECT ON DATABASE backupdb to backup; "
+                "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
+                "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
+                "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
+                "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
+            )
+        else:
+            node.safe_psql(
+                'backupdb',
+                "REVOKE ALL ON DATABASE backupdb from PUBLIC; "
+                "REVOKE ALL ON SCHEMA public from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; "
+                "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; "
+                "REVOKE ALL ON SCHEMA information_schema from PUBLIC; "
+                "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; "
+                "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; "
+                "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; "
+                "CREATE ROLE backup WITH LOGIN REPLICATION; "
+                "GRANT CONNECT ON DATABASE backupdb to backup; "
+                "GRANT USAGE ON SCHEMA pg_catalog TO backup; "
+                "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; "
+                "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack
+                "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_wal_replay_lsn() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; "
+                "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;"
+            )
+
+        if self.ptrack:
+            for fname in [
+                    'oideq(oid, oid)',
+                    'ptrack_version()',
+                    'pg_ptrack_clear()',
+                    'pg_ptrack_control_lsn()',
+                    'pg_ptrack_get_and_clear_db(oid, oid)',
+                    'pg_ptrack_get_and_clear(oid, oid)',
+                    'pg_ptrack_get_block_2(oid, oid, oid, bigint)']:
+                # try:
+                node.safe_psql(
+                    "backupdb",
+                    "GRANT EXECUTE ON FUNCTION pg_catalog.{0} "
+                    "TO backup".format(fname))
+                # except:
+                #     pass
 
         # FULL backup
         self.backup_node(
@@ -1584,12 +1635,13 @@ def test_backup_with_least_privileges_role(self):
             datname='backupdb', options=['--stream', '-U', 'backup'])
 
         # PTRACK
-        # self.backup_node(
-        #     backup_dir, 'node', node, backup_type='ptrack',
-        #     datname='backupdb', options=['-U', 'backup'])
-        # self.backup_node(
-        #     backup_dir, 'node', node, backup_type='ptrack',
-        #     datname='backupdb', options=['--stream', '-U', 'backup'])
+        if self.ptrack:
+            self.backup_node(
+                backup_dir, 'node', node, backup_type='ptrack',
+                datname='backupdb', options=['-U', 'backup'])
+            self.backup_node(
+                backup_dir, 'node', node, backup_type='ptrack',
+                datname='backupdb', options=['--stream', '-U', 'backup'])
 
         # Clean after yourself
         self.del_test_dir(module_name, fname)

tests/checkdb.py

@@ -229,7 +229,7 @@ def test_basic_checkdb_amcheck_only_sanity(self):
         # create two databases
         node.safe_psql("postgres", "create database db1")
         try:
-           node.safe_psql(
+            node.safe_psql(
                "db1",
                "create extension amcheck")
         except QueryException as e:

tests/compatibility.py

@@ -217,15 +217,20 @@ def test_backward_compatibility_delta(self):
     # @unittest.skip("skip")
     def test_backward_compatibility_ptrack(self):
         """Description in jira issue PGPRO-434"""
+
+        if not self.ptrack:
+            return unittest.skip('Skipped because ptrack support is disabled')
+
         fname = self.id().split('.')[3]
         backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
         node = self.make_simple_node(
             base_dir=os.path.join(module_name, fname, 'node'),
             set_replication=True,
             initdb_params=['--data-checksums'],
             pg_options={
-                'autovacuum': 'off'}
-            )
+                'autovacuum': 'off',
+                'ptrack_enable': 'on'})
+
         self.init_pb(backup_dir, old_binary=True)
         self.show_pb(backup_dir)
 
@@ -262,7 +267,7 @@ def test_backward_compatibility_ptrack(self):
             pgdata_restored = self.pgdata_content(node_restored.data_dir)
             self.compare_pgdata(pgdata, pgdata_restored)
 
-        # Delta BACKUP with old binary
+        # ptrack BACKUP with old binary
         pgbench = node.pgbench(
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
@@ -272,7 +277,7 @@ def test_backward_compatibility_ptrack(self):
         pgbench.stdout.close()
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='delta',
+            backup_dir, 'node', node, backup_type='ptrack',
             old_binary=True)
 
         if self.paranoia:
@@ -287,7 +292,7 @@ def test_backward_compatibility_ptrack(self):
             pgdata_restored = self.pgdata_content(node_restored.data_dir)
             self.compare_pgdata(pgdata, pgdata_restored)
 
-        # Delta BACKUP with new binary
+        # Ptrack BACKUP with new binary
         pgbench = node.pgbench(
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
@@ -297,7 +302,7 @@ def test_backward_compatibility_ptrack(self):
         pgbench.stdout.close()
 
         self.backup_node(
-            backup_dir, 'node', node, backup_type='delta')
+            backup_dir, 'node', node, backup_type='ptrack')
 
         if self.paranoia:
             pgdata = self.pgdata_content(node.data_dir)