Fix flakey test isolation2/unlogged_heap_tables due to primary/mirror switch by fts. (#7338)

unlogged_heap_tables test occasionally fails since during primary restarting
there is a possibility that fts switches the mirrors to primaries.  For
unlogged table, there is no data on the mirrors so select query could return
different results if there is or there is no switch, thus makes the test not
deterministic.

We disable fts probe in this test using gp fault injector to fix this issue.
Co-authored-by: NHaozhou Wang <hawang@pivotal.io>
Reviewed-by: NAshwin Agrawal <aagrawal@pivotal.io>
Reviewed-by: NJimmy Yih <jyih@pivotal.io>

src/test/isolation2/expected/unlogged_appendonly_tables.out

@@ -6,6 +6,29 @@ CREATE
 create unlogged table unlogged_appendonly_table_managers ( id int, name text ) with ( appendonly=true ) distributed by (id);
 CREATE
 
+-- skip FTS probes to make the test deterministic.
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+CREATE
+SELECT gp_inject_fault_infinite('fts_probe', 'skip', 1);
+ gp_inject_fault_infinite 
+--------------------------
+ t                        
+(1 row)
+SELECT gp_request_fts_probe_scan();
+ gp_request_fts_probe_scan 
+---------------------------
+ t                         
+(1 row)
+SELECT gp_request_fts_probe_scan();
+ gp_request_fts_probe_scan 
+---------------------------
+ t                         
+(1 row)
+SELECT gp_wait_until_triggered_fault('fts_probe', 1, 1);
+ gp_wait_until_triggered_fault 
+-------------------------------
+ t                             
+(1 row)
 
 -- expect: insert/update/select works
 insert into unlogged_appendonly_table_managers values (1, 'Joe');
@@ -79,3 +102,8 @@ UPDATE 1
 5: drop table unlogged_appendonly_table_managers;
 DROP
 
+SELECT gp_inject_fault('fts_probe', 'reset', 1);
+ gp_inject_fault 
+-----------------
+ t               
+(1 row)

src/test/isolation2/expected/unlogged_heap_tables.out

@@ -6,6 +6,29 @@ CREATE
 create unlogged table unlogged_heap_table_managers ( id int, name text ) distributed by (id);
 CREATE
 
+-- skip FTS probes to make the test deterministic.
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+CREATE
+SELECT gp_inject_fault_infinite('fts_probe', 'skip', 1);
+ gp_inject_fault_infinite 
+--------------------------
+ t                        
+(1 row)
+SELECT gp_request_fts_probe_scan();
+ gp_request_fts_probe_scan 
+---------------------------
+ t                         
+(1 row)
+SELECT gp_request_fts_probe_scan();
+ gp_request_fts_probe_scan 
+---------------------------
+ t                         
+(1 row)
+SELECT gp_wait_until_triggered_fault('fts_probe', 1, 1);
+ gp_wait_until_triggered_fault 
+-------------------------------
+ t                             
+(1 row)
 
 -- expect: insert/update/select works
 insert into unlogged_heap_table_managers values (1, 'Joe');
@@ -59,3 +82,8 @@ UPDATE 1
 5: drop table unlogged_heap_table_managers;
 DROP
 
+SELECT gp_inject_fault('fts_probe', 'reset', 1);
+ gp_inject_fault 
+-----------------
+ t               
+(1 row)

src/test/isolation2/sql/unlogged_appendonly_tables.sql

@@ -9,6 +9,12 @@ create unlogged table unlogged_appendonly_table_managers (
 	appendonly=true
 ) distributed by (id);
 
+-- skip FTS probes to make the test deterministic.
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+SELECT gp_inject_fault_infinite('fts_probe', 'skip', 1);
+SELECT gp_request_fts_probe_scan();
+SELECT gp_request_fts_probe_scan();
+SELECT gp_wait_until_triggered_fault('fts_probe', 1, 1);
 
 -- expect: insert/update/select works
 insert into unlogged_appendonly_table_managers values (1, 'Joe');
@@ -47,3 +53,4 @@ select clean_restart_primary_segments_containing_data_for('unlogged_appendonly_t
 -- expect: drop table succeeds
 5: drop table unlogged_appendonly_table_managers;
 
+SELECT gp_inject_fault('fts_probe', 'reset', 1);

src/test/isolation2/sql/unlogged_heap_tables.sql

@@ -7,6 +7,12 @@ create unlogged table unlogged_heap_table_managers (
 	name text
 ) distributed by (id);
 
+-- skip FTS probes to make the test deterministic.
+CREATE EXTENSION IF NOT EXISTS gp_inject_fault;
+SELECT gp_inject_fault_infinite('fts_probe', 'skip', 1);
+SELECT gp_request_fts_probe_scan();
+SELECT gp_request_fts_probe_scan();
+SELECT gp_wait_until_triggered_fault('fts_probe', 1, 1);
 
 -- expect: insert/update/select works
 insert into unlogged_heap_table_managers values (1, 'Joe');
@@ -41,3 +47,4 @@ select clean_restart_primary_segments_containing_data_for('unlogged_heap_table_m
 -- expect: drop table succeeds
 5: drop table unlogged_heap_table_managers;
 
+SELECT gp_inject_fault('fts_probe', 'reset', 1);