From 714a8375a39ea373129b72b3bacede9d72bb3543 Mon Sep 17 00:00:00 2001 From: Abhijit Subramanya Date: Wed, 11 Jan 2017 17:43:51 -0800 Subject: [PATCH] Fix vacuum and access method tests. - Update the answer files for vacuum related tests. - Update the SQL used to increase the age of the table in mpp24168 test. - Remove tests that use gp_filedump. --- .../test_appendonly_checksum_scenario.py | 66 -------------- .../test_sanity_deltacompression.py | 90 ------------------- .../storage/vacuum/mpp24168/test_mpp24168.py | 2 +- .../vacuum_while_reindex_heap_btree_toast.ans | 2 +- .../vacuum_while_reindex_heap_btree_toast.sql | 2 +- .../vacuum/scenario/lockrelease/Makefile | 3 + .../scenario/xml/catalog_master_error.xml | 2 +- .../xml/heap_crash_after_truncate.xml | 4 +- .../xml/heap_crash_before_truncate.xml | 4 +- .../xml/heap_mastercrash_before_truncate.xml | 2 +- .../xml/heap_mastererror_before_truncate.xml | 2 +- .../xml/heap_update_crash_before_truncate.xml | 4 +- .../vacuum/simple/sql/vacuum_full_ao.ans | 13 --- .../vacuum/simple/sql/vacuum_full_ao.sql | 2 - 14 files changed, 15 insertions(+), 183 deletions(-) delete mode 100644 src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/deltacompression/test_sanity_deltacompression.py diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/appendonly_checksum/test_appendonly_checksum_scenario.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/appendonly_checksum/test_appendonly_checksum_scenario.py index 1b2754ff5d..036a5cfa34 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/appendonly_checksum/test_appendonly_checksum_scenario.py +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/appendonly_checksum/test_appendonly_checksum_scenario.py @@ -208,57 +208,6 @@ class AppendonlyChecksumTestCase(ScenarioTestCase, MPPTestCase): Command("Restore data-file", cmd).run(validateAfter=True) self.select_table(verify_sql,noncorrupted_checksum=True) - def test_aochecksum_size(self): - ''' - The following test verifies that the header length is calculated properly when the checksum is on & off - for small, large and bulk-dense header content. - - PARAMETERS (via data_provider): - 0. Test Name - 1. Sql file to create scenario. - 2. Expected length of the header - 3. Expected type of the header - - STEPS : - 1. Create table of required type of header (bulk, large header_content or small header_content (ao/co) - 2. Find the data file for this relationship - 3. Get the gp_filedump of the data file - 4. Verify expected length of the header - 5. Verify expected type of the header - - @data_provider data_types_provider_aochecksum_size - ''' - - tab_name = self.test_data[0] - chksum_flag = self.test_data[1][0] - header_size = self.test_data[1][1] - header_type = self.test_data[1][2] - tinctest.logger.info('=======================================') - tinctest.logger.info('Starting Test %s' % tab_name) - tinctest.logger.info('Table Name %s' % tab_name) - tinctest.logger.info('chksum_flag %s' % chksum_flag ) - tinctest.logger.info('header_size %s' % header_size) - tinctest.logger.info('header_type %s' % header_type) - tinctest.logger.info('=======================================') - self.create_table(tab_name) - (host, db_path) = self.gpfile.get_host_and_db_path(self.dbname) - tinctest.logger.info('Hostname=%s data_directory=%s' %(host,db_path)) - tinctest.logger.info( "tab_name : %s chksum_flag: %s header_size: %s header_type: %s" % (tab_name,chksum_flag,header_size,header_type)) - file_list = self.gpfile.get_relfile_list(self.dbname, tab_name, db_path, host) - for i in range(0, len(file_list)): - tinctest.logger.info('Getting checksum info for table %s with relfilenode %s' % (tab_name, file_list[i])) - flag='' - if chksum_flag == 'on': - flag=' -M' - if tab_name.endswith('ao'): - flag=flag+' -O row ' - has_headerlen = self.gpfile.check_in_filedump(db_path, host, file_list[i], header_size,flag) - has_headertype = self.gpfile.check_in_filedump(db_path, host, file_list[i], header_type,flag) - if not has_headerlen: - raise Exception ("Checksum validation failed for table %s with relfilenode: %s because header length is not %s" % (tab_name,file_list[i],header_size)) - if not has_headertype: - raise Exception ("Checksum validation failed for table %s with relfilenode: %s because header type is not %s" % (tab_name,file_list[i],header_type)) - @tinctest.dataProvider('data_provider_for_checksum_corruption') def test_data_provider_for_checksum_corruption(): data = { @@ -271,18 +220,3 @@ def test_data_provider_for_checksum_corruption(): "appendonly_verify_block_checksums_co":['appendonly_verify_block_checksums_co',CONST.EOF,CONST.FIND_CHAR] } return data - -@tinctest.dataProvider('data_types_provider_aochecksum_size') -def test_data_provider_aochecksum_size(): - data = { - "chksum_on_header_bulkdense_co":[CONST.CHKSUM_ON,CONST.BULKDENSE_HEADER_LEN_WITH_CHKSUM_ON,CONST.BULKDENSE_HEADER_TYPE], - "chksum_off_header_bulkdense_co":[CONST.CHKSUM_OFF,CONST.BULKDENSE_HEADER_LEN_WITH_CHKSUM_OFF,CONST.BULKDENSE_HEADER_TYPE], - "chksum_off_header_sml_ao":[CONST.CHKSUM_OFF,CONST.SMALL_HEADER_LEN_WITH_CHKSUM_OFF,CONST.SMALL_HEADER_TYPE], - "chksum_off_header_sml_co":[CONST.CHKSUM_OFF,CONST.SMALL_HEADER_LEN_WITH_CHKSUM_OFF,CONST.SMALL_HEADER_TYPE], - "chksum_on_header_sml_ao":[CONST.CHKSUM_ON,CONST.SMALL_HEADER_LEN_WITH_CHKSUM_ON,CONST.SMALL_HEADER_TYPE], - "chksum_on_header_sml_co":[CONST.CHKSUM_ON,CONST.SMALL_HEADER_LEN_WITH_CHKSUM_ON,CONST.SMALL_HEADER_TYPE], - "chksum_on_header_large_co":[CONST.CHKSUM_ON,CONST.LARGE_HEADER_LEN_WITH_CHKSUM_ON,CONST.LARGE_HEADER_TYPE], - "chksum_off_header_large_co":[CONST.CHKSUM_OFF,CONST.LARGE_HEADER_LEN_WITH_CHKSUM_OFF,CONST.LARGE_HEADER_TYPE] - } - return data - diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/deltacompression/test_sanity_deltacompression.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/deltacompression/test_sanity_deltacompression.py deleted file mode 100644 index 0c0a69886f..0000000000 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/access_methods/deltacompression/test_sanity_deltacompression.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Copyright (C) 2004-2015 Pivotal Software, Inc. All rights reserved. - -This program and the accompanying materials are made available under -the terms of the under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import os -import tinctest -from mpp.models import SQLTestCase -from mpp.models import MPPTestCase -from tinctest.models.scenario import ScenarioTestCase -from mpp.gpdb.tests.storage.lib.gp_filedump import GpfileTestCase -''' -Test for Delta Compression -''' -@tinctest.skipLoading('scenario') -class DeltaCompressionTestCase(SQLTestCase): - """ - @gucs gp_create_table_random_default_distribution=off - - @dbname deltagp - @product_version gpdb: [4.3.3.0-] - """ - - sql_dir = 'sql_gp/' - ans_dir = 'expected/' - out_dir = 'output/' - - -@tinctest.skipLoading('scenario') -class DeltaHelper(MPPTestCase): - - def gp_filedump_check(self, checksum=False): - gpfile = GpfileTestCase() - dbname=os.environ.get('PGDATABASE') - if checksum == True: - flag = " -M " - else: - flag = " " - table_list = {"delta_1_byte":"deltas_size: 1","delta_2_byte":"deltas_size: 2", - "delta_3_byte":"deltas_size: 3","delta_4_byte":"deltas_size: 4", - "delta_neg_1_byte":"deltas_size: 1","delta_neg_2_byte":"deltas_size: 2", - "delta_neg_3_byte":"deltas_size: 3","delta_neg_4_byte":"deltas_size: 4"} - - (host, db_path) = gpfile.get_host_and_db_path(dbname) - # Cases where we expect a delta - for tablename in table_list: - file_list = gpfile.get_relfile_list(dbname, tablename, db_path, host) - for i in range(1, len(file_list)): - tinctest.logger.info('Running compression check for table %s relfilenode %s' % (tablename, file_list[i])) - has_delta = gpfile.check_in_filedump(db_path, host, file_list[i], 'HAS_DELTA_COMPRESSION', flag) - is_byte = gpfile.check_in_filedump(db_path, host, file_list[i], table_list[tablename], flag) - if not has_delta or not is_byte: - raise Exception ("Delta Compression not applied to relation with relfilenode: %s " % file_list[i]) - - # Cases where we are not expecting a delta - tablename = 'delta_more_4_byte' - file_list = gpfile.get_relfile_list(dbname, tablename, db_path, host) - for i in range(1, len(file_list)): - tinctest.logger.info('Running compression check for table %s relfilenode %s' % (tablename, file_list[i])) - delta = gpfile.check_in_filedump(db_path, host, file_list[i], 'HAS_DELTA_COMPRESSION', flag) - if delta: - raise Exception("Delta Compression not expected for %s" % file_list[i]) - - -class DeltaScenarioTestCase(ScenarioTestCase, MPPTestCase): - """ - @gucs gp_create_table_random_default_distribution=off - """ - - def test_delta_byte_check_post_434(self): - ''' - @product_version gpdb: [4.3.4.0-] - ''' - test_list_1 = [] - test_list_1.append(('mpp.gpdb.tests.storage.access_methods.deltacompression.test_sanity_deltacompression.DeltaCompressionTestCase')) - test_list_1.append(('mpp.gpdb.tests.storage.access_methods.deltacompression.test_sanity_deltacompression.DeltaHelper.gp_filedump_check', {'checksum' : True})) - self.test_case_scenario.append(test_list_1, serial=True) - diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/mpp24168/test_mpp24168.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/mpp24168/test_mpp24168.py index 9a4fc89001..ac1bb1d03f 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/mpp24168/test_mpp24168.py +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/mpp24168/test_mpp24168.py @@ -37,7 +37,7 @@ class RelfrozenxidUpdateTestCase(MPPTestCase): #doing multiple selects to increase the age of the table for x in range(0, 150): - PSQL.run_sql_command("select * from test_table", flags ='-q -t') + PSQL.run_sql_command("select age(relfrozenxid) from gp_dist_random('pg_class') where relname = 'test_table'", flags ='-q -t') PSQL.run_sql_command("drop table if exists table_master; \ diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/expected/vacuum_while_reindex_heap_btree_toast.ans b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/expected/vacuum_while_reindex_heap_btree_toast.ans index 7349892056..7c90b2f395 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/expected/vacuum_while_reindex_heap_btree_toast.ans +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/expected/vacuum_while_reindex_heap_btree_toast.ans @@ -14,7 +14,7 @@ COMMIT VACUUM 2: COMMIT; COMMIT -3: SELECT COUNT(*) FROM reindex_toast_heap WHERE a = 1500; +3: SELECT COUNT(*) FROM reindex_toast_heap WHERE a = '1500'; count ----- 0 diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/sql/vacuum_while_reindex_heap_btree_toast.sql b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/sql/vacuum_while_reindex_heap_btree_toast.sql index 7dee614571..bc620b3521 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/sql/vacuum_while_reindex_heap_btree_toast.sql +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/reindex/concurrency/sql/vacuum_while_reindex_heap_btree_toast.sql @@ -8,6 +8,6 @@ DELETE FROM reindex_toast_heap WHERE b % 4 = 0 ; 1: COMMIT; 2<: 2: COMMIT; -3: SELECT COUNT(*) FROM reindex_toast_heap WHERE a = 1500; +3: SELECT COUNT(*) FROM reindex_toast_heap WHERE a = '1500'; 3: INSERT INTO reindex_toast_heap VALUES (0); 3: SELECT 1 AS relfilenode_same_on_all_segs from gp_dist_random('pg_class') WHERE relname = 'idx_btree_reindex_toast_heap' GROUP BY relfilenode having count(*) = (SELECT count(*) FROM gp_segment_configuration WHERE role='p' AND content > -1); diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/lockrelease/Makefile b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/lockrelease/Makefile index 748d7bdac1..14ca7461c2 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/lockrelease/Makefile +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/lockrelease/Makefile @@ -1,6 +1,9 @@ MODULES = lockrelease DATA_built = install.sql +override CC:=gcc +override CFLAGS:=-m64 + PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/xml/catalog_master_error.xml b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/xml/catalog_master_error.xml index 6f59125167..28e1e78052 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/xml/catalog_master_error.xml +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/vacuum/scenario/xml/catalog_master_error.xml @@ -490,7 +490,7 @@ INSERT 0 99 ]]>