From fabb8d586bd19db1649a16ee7f3fdc2f023c1ee6 Mon Sep 17 00:00:00 2001 From: Ashwin Agrawal Date: Fri, 28 Oct 2016 19:17:30 -0700 Subject: [PATCH] Make uao fault injector tests deterministic. Replace sleeps with deterministic check for database operational, after injecting PANIC faults. --- .../test_uao_fault_inject.py | 123 +++++++----------- src/test/tinc/tincrepo/mpp/lib/PSQL.py | 18 +++ 2 files changed, 66 insertions(+), 75 deletions(-) diff --git a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/uao/uao_faultinjection/test_uao_fault_inject.py b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/uao/uao_faultinjection/test_uao_fault_inject.py index 86bee211bc..8a09b7d622 100644 --- a/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/uao/uao_faultinjection/test_uao_fault_inject.py +++ b/src/test/tinc/tincrepo/mpp/gpdb/tests/storage/uao/uao_faultinjection/test_uao_fault_inject.py @@ -25,7 +25,6 @@ import unittest2 as unittest import os import subprocess import sys -from time import sleep @unittest.skipIf(os.environ.get('BUILD_TYPE') == 'gcov', "Skip for Instrumented Build") @@ -84,10 +83,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -114,9 +111,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -143,9 +139,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -172,9 +167,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -201,9 +195,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -255,10 +248,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -285,9 +276,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -314,9 +304,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_master_reset) gpfaultinjector.run() @@ -343,9 +332,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -372,9 +360,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -401,9 +388,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -430,9 +416,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -459,9 +444,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -488,9 +472,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -517,9 +500,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -546,9 +528,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -575,9 +556,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -604,9 +584,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -633,9 +612,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -662,9 +640,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -691,9 +668,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -720,9 +696,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -749,9 +724,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() @@ -778,9 +752,8 @@ class UAO_FaultInjection_TestCase(MPPTestCase): PSQL.run_sql_file(sql_file1, out_file=out_file1) result1 = Gpdiff.are_files_equal(out_file1, ans_file1) - # The connection is accepted before it is actually ready, leading to panic messages - # Therefore we have to sleep here. - sleep(5) + + PSQL.wait_for_database_up(); gpfaultinjector = Command(cmd_type, set_fault_in_seg_reset) gpfaultinjector.run() diff --git a/src/test/tinc/tincrepo/mpp/lib/PSQL.py b/src/test/tinc/tincrepo/mpp/lib/PSQL.py index 2c5e7a63b0..9cdb0b8606 100644 --- a/src/test/tinc/tincrepo/mpp/lib/PSQL.py +++ b/src/test/tinc/tincrepo/mpp/lib/PSQL.py @@ -547,3 +547,21 @@ class PSQL(Command): if 'command_found_' + dbname in output: return True return False + + @staticmethod + def wait_for_database_up(): + ''' + Wait till the system is up, as master may take some time + to come back after FI crash. + ''' + down = True + results = {'rc':0, 'stdout':'', 'stderr':''} + for i in range(60): + res = PSQL.run_sql_command('select count(*) from gp_dist_random(\'gp_id\');', results=results) + if results['rc'] == 0: + down = False + break + time.sleep(1) + + if down: + raise PSQLException('database has not come up') -- GitLab