Make pg_terminate_backend test more reliable

This test has failed at least once due to the terminate query being executed
before the to be terminated 'create table' statement.  This was evident from
master logs.  The commit makes it more reliable by injecting a fault and
waiting for the fault to be trigggered before executing pg_terminate_backend().
As a side benefit, we no longer need to create any additional table.

src/backend/access/heap/heapam.c

@@ -2363,6 +2363,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid,
 
 	gp_expand_protect_catalog_changes(relation);
 
+#ifdef FAULT_INJECTOR
+	FaultInjector_InjectFaultIfSet(HeapInsert, DDLNotSpecified, "",
+								   RelationGetRelationName(relation));
+#endif
+
 	/*
 	 * Fill in tuple header fields, assign an OID, and toast the tuple if
 	 * necessary.

src/include/utils/faultinjector_lists.h

@@ -259,6 +259,8 @@ FI_IDENT(XLogAoInsert, "xlog_ao_insert")
 FI_IDENT(InsideMoveDbTransaction, "inside_move_db_transaction")
 /* inject fault just after calculating redo record and before committing checkpoint record */
 FI_IDENT(CheckpointAfterRedoCalculated, "checkpoint_after_redo_calculated")
+/* inject fault at the beginning of heap insert */
+FI_IDENT(HeapInsert, "heap_insert")
 #endif
 
 /*

src/test/isolation2/expected/pg_terminate_backend.out

-create table foo as select i a, i b from generate_series(1, 10) i;
-CREATE 10
+1:create table terminate_backend_t (a int) distributed by (a);
+CREATE
 
--- expect this query terminated by 'test pg_terminate_backend'
-1&:create temp table t as select count(*) from foo where pg_sleep(20) is null;  <waiting ...>
+-- fault on seg1 to block insert command into terminate_backend_t table
+select gp_inject_fault('heap_insert', 'infinite_loop', '', '', 'terminate_backend_t', 1, 1, 0, dbid) from gp_segment_configuration where content = 1 and role = 'p';
+ gp_inject_fault 
+-----------------
+ t               
+(1 row)
+
+-- expect this command to be terminated by 'test pg_terminate_backend'
+1&: insert into terminate_backend_t values (1);  <waiting ...>
+
+select gp_wait_until_triggered_fault('heap_insert', 1, dbid) from gp_segment_configuration where content = 1 and role = 'p';
+ gp_wait_until_triggered_fault 
+-------------------------------
+ t                             
+(1 row)
 
 -- extract the pid for the previous query
-SELECT pg_terminate_backend(pid,'test pg_terminate_backend') FROM pg_stat_activity WHERE query like 'create temp table t as select%' ORDER BY pid LIMIT 1;
+SELECT pg_terminate_backend(pid,'test pg_terminate_backend') FROM pg_stat_activity WHERE query like 'insert into terminate_backend_t%' ORDER BY pid LIMIT 1;
  pg_terminate_backend 
 ----------------------
  t                    
@@ -19,8 +32,14 @@ server closed the connection unexpectedly
 	before or while processing the request.
 
 -- query backend to ensure no PANIC on postmaster
-select count(*) from foo;
- count 
--------
- 10    
+select gp_inject_fault('heap_insert', 'reset', dbid) from gp_segment_configuration where content = 1 and role = 'p';
+ gp_inject_fault 
+-----------------
+ t               
 (1 row)
+
+-- the table should be empty if insert was terminated
+select * from terminate_backend_t;
+ a 
+---
+(0 rows)

src/test/isolation2/sql/pg_terminate_backend.sql

-create table foo as select i a, i b from generate_series(1, 10) i;
+1:create table terminate_backend_t (a int) distributed by (a);
 
--- expect this query terminated by 'test pg_terminate_backend'
-1&:create temp table t as select count(*) from foo where pg_sleep(20) is null;
+-- fault on seg1 to block insert command into terminate_backend_t table
+select gp_inject_fault('heap_insert', 'infinite_loop', '', '',
+   'terminate_backend_t', 1, 1, 0, dbid) from gp_segment_configuration
+   where content = 1 and role = 'p';
+
+-- expect this command to be terminated by 'test pg_terminate_backend'
+1&: insert into terminate_backend_t values (1);
+
+select gp_wait_until_triggered_fault('heap_insert', 1, dbid)
+from gp_segment_configuration where content = 1 and role = 'p';
 
 -- extract the pid for the previous query
 SELECT pg_terminate_backend(pid,'test pg_terminate_backend')
-FROM pg_stat_activity WHERE query like 'create temp table t as select%' ORDER BY pid LIMIT 1;
+FROM pg_stat_activity WHERE query like 'insert into terminate_backend_t%'
+ORDER BY pid LIMIT 1;
 
 -- EXPECT: session 1 terminated with 'test pg_terminate_backend'
 1<:
 
 -- query backend to ensure no PANIC on postmaster
-select count(*) from foo;
+select gp_inject_fault('heap_insert', 'reset', dbid)
+   from gp_segment_configuration
+   where content = 1 and role = 'p';
+
+-- the table should be empty if insert was terminated
+select * from terminate_backend_t;