Update missing_xlog to make it more deterministic

Before we shutdown the mirror, we create a checkpoint and also do an
empty transaction to ensure the restart point is created on mirror.

This will speedup the mirror recovery because we don't have to reply all
the xlog records from the beginning.

Author: Xin Zhang <xzhang@pivotal.io>
Author: Taylor Vesely <tvesely@pivotal.io>

src/test/walrep/expected/missing_xlog.out

@@ -27,6 +27,26 @@ returns text as $$
 	
 	return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).replace('.', '')
 $$ language plpythonu;
+create or replace function wait_for_replication_replay (retries int) returns bool as
+$$
+declare
+	i int;
+begin
+	i := 0;
+	loop
+		if (select count(*) from gp_stat_replication) =
+			(select count(*) from gp_stat_replication
+				where replay_location = sent_location) then
+			return true;
+		end if;
+		if i >= retries then
+			return false;
+		end if;
+		perform pg_sleep(0.1);
+		i := i + 1;
+	end loop;
+end;
+$$ language plpgsql;
 create or replace function wait_for_replication_error (expected_error text, segment_id int, retries int) returns bool as
 $$
 declare
@@ -45,6 +65,15 @@ begin
 	end loop;
 end;
 $$ language plpgsql;
+-- checkpoint to ensure clean xlog replication before bring down mirror
+checkpoint;
+begin;end;
+select wait_for_replication_replay(200);
+ wait_for_replication_replay 
+-----------------------------
+ t
+(1 row)
+
 -- stop a mirror 
 select pg_ctl((select fselocation from gp_segment_configuration c, pg_filespace_entry f where c.role='m' and c.content=0 and c.dbid = f.fsedbid), 'stop', NULL, NULL);
                 pg_ctl                

src/test/walrep/sql/missing_xlog.sql

@@ -30,6 +30,27 @@ returns text as $$
 	return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).replace('.', '')
 $$ language plpythonu;
 
+create or replace function wait_for_replication_replay (retries int) returns bool as
+$$
+declare
+	i int;
+begin
+	i := 0;
+	loop
+		if (select count(*) from gp_stat_replication) =
+			(select count(*) from gp_stat_replication
+				where replay_location = sent_location) then
+			return true;
+		end if;
+		if i >= retries then
+			return false;
+		end if;
+		perform pg_sleep(0.1);
+		i := i + 1;
+	end loop;
+end;
+$$ language plpgsql;
+
 create or replace function wait_for_replication_error (expected_error text, segment_id int, retries int) returns bool as
 $$
 declare
@@ -49,6 +70,11 @@ begin
 end;
 $$ language plpgsql;
 
+-- checkpoint to ensure clean xlog replication before bring down mirror
+checkpoint;
+begin;end;
+select wait_for_replication_replay(200);
+
 -- stop a mirror 
 select pg_ctl((select fselocation from gp_segment_configuration c, pg_filespace_entry f where c.role='m' and c.content=0 and c.dbid = f.fsedbid), 'stop', NULL, NULL);