isolation2: Fix race condition in mirror_promotion

Force the mirror to create a restartpoint, and as a side-effect replay
the DROP TABLESPACE DDL before removing the tablespace directory.
Co-authored-by: NSoumyadeep Chakraborty <sochakraborty@pivotal.io>

src/test/isolation2/expected/segwalrep/mirror_promotion.out

@@ -136,10 +136,19 @@ select content, preferred_role, role, status, mode from gp_segment_configuration
 -- set GUCs to speed-up the test
 !\retcode gpconfig -r gp_fts_probe_retries --masteronly;
 -- start_ignore
+
 -- end_ignore
 (exited with code 0)
 !\retcode gpconfig -r gp_fts_probe_timeout --masteronly;
 -- start_ignore
+
+-- end_ignore
+(exited with code 0)
+
+-- Set GUC to force tablespace drop replay to complete on mirror before
+-- removing the directory
+!\retcode gpconfig -c create_restartpoint_on_ckpt_record_replay -v on;
+-- start_ignore
 -- end_ignore
 (exited with code 0)
 !\retcode gpstop -u;
@@ -156,6 +165,9 @@ select content, preferred_role, role, status, mode from gp_segment_configuration
 
 -- create tablespace to test if it works with gprecoverseg -F (pg_basebackup)
 !\retcode mkdir /tmp/mirror_promotion_tablespace_loc;
+-- start_ignore
+
+-- end_ignore
 (exited with code 0)
 create tablespace mirror_promotion_tablespace location '/tmp/mirror_promotion_tablespace_loc';
 CREATE
@@ -172,7 +184,26 @@ drop table mirror_promotion_tblspc_heap_table;
 DROP
 drop tablespace mirror_promotion_tablespace;
 DROP
+-- Force the mirror to replay the drop before moving forward with tablespace
+-- directory deletion
+checkpoint;
+CHECKPOINT
 !\retcode rm -rf /tmp/mirror_promotion_tablespace_loc;
+-- start_ignore
+
+-- end_ignore
+(exited with code 0)
+
+-- Reset create_restartpoint_on_ckpt_record_replay guc
+!\retcode gpconfig -r create_restartpoint_on_ckpt_record_replay;
+-- start_ignore
+
+-- end_ignore
+(exited with code 0)
+!\retcode gpstop -u;
+-- start_ignore
+
+-- end_ignore
 (exited with code 0)
 
 -- loop while segments come in sync

src/test/isolation2/sql/segwalrep/mirror_promotion.sql

@@ -83,6 +83,10 @@ where content = 0;
 -- set GUCs to speed-up the test
 !\retcode gpconfig -r gp_fts_probe_retries --masteronly;
 !\retcode gpconfig -r gp_fts_probe_timeout --masteronly;
+
+-- Set GUC to force tablespace drop replay to complete on mirror before
+-- removing the directory
+!\retcode gpconfig -c create_restartpoint_on_ckpt_record_replay -v on;
 !\retcode gpstop -u;
 
 -- -- wait for content 0 (earlier mirror, now primary) to finish the promotion
@@ -98,8 +102,15 @@ create table mirror_promotion_tblspc_heap_table (a int) tablespace mirror_promot
 
 drop table mirror_promotion_tblspc_heap_table;
 drop tablespace mirror_promotion_tablespace;
+-- Force the mirror to replay the drop before moving forward with tablespace
+-- directory deletion
+checkpoint;
 !\retcode rm -rf /tmp/mirror_promotion_tablespace_loc;
 
+-- Reset create_restartpoint_on_ckpt_record_replay guc
+!\retcode gpconfig -r create_restartpoint_on_ckpt_record_replay;
+!\retcode gpstop -u;
+
 -- loop while segments come in sync
 do $$
 begin /* in func */