Recursively create tablespace directories if they do not exist but we need...

Recursively create tablespace directories if they do not exist but we need them when re-redoing some tablespace related xlogs (e.g. database create with a tablespace) on mirror.

It is observed many time that gp_replica_check test fails because some mirror nodes
can not be brought up before testing recently. The related log looks like this:

2019-04-17 14:52:14.951 CST [23030] FATAL:  could not create directory "pg_tblspc/65546/PG_12_201904072/65547": No such file or directory
2019-04-17 14:52:14.951 CST [23030] CONTEXT:  WAL redo at 0/3011650 for Database/CREATE: copy dir 1663/1 to 65546/65547

That is because some mirror nodes can not be recovered after previous testing,
not due to gp_replica_check itself. The root cause is that tablespace recovery
related. Pengzhou Tang and Hao Wu digged that intially and kindly found a mini
repro as below.

run on shell:
  rm -rf /tmp/some_isolation2_pg_basebackup_tablespace
  mkdir -p /tmp/some_isolation2_pg_basebackup_tablespace

copy and run the below sql on psql client:
  drop tablespace if exists some_isolation2_pg_basebackup_tablespace;
  create tablespace some_isolation2_pg_basebackup_tablespace location '/tmp/some_isolation2_pg_basebackup_tablespace';
  \!gpstop -ra -M fast;
  drop database if exists some_database_with_tablespace;
  create database some_database_with_tablespace tablespace some_isolation2_pg_basebackup_tablespace;
  drop database some_database_with_tablespace;
  drop tablespace some_isolation2_pg_basebackup_tablespace;
  \!gpstop -ra -M immediate;

The root cause is on mirror after drop database & drop tablespace, 'immediate'
stop causes the pg_control file not up-to-date with latest redo start lsn (this
is allowed), when the node restarts, it re-redoes 'create database
some_database_with_tablespace tablespace
some_isolation2_pg_basebackup_tablespace' but the tablespace directories have
been deleted in previous redoing.

The 'could not create directory' error could happen on re-redoing create table
in a tablespace also. We've seen this case on the ci environment, but that is
because missing of a get_parent_directory() call in the 'create two parents'
code block in TablespaceCreateDbspace(). Changing it to a simpler call
pg_mkdir_p() instead.

Also it seems that the src_path could be missing also in dbase_redo() for the
example below. For example re-redoing at the alter step since tbs1 directory is
deleted in later 'drop tablespace tbs1'.
  alter database db1 set tablespace tbs2;
  drop tablespace tbs1;

There is discussion on upstream about this,
  https://www.postgresql.org/message-id/flat/CAEET0ZGx9AvioViLf7nbR_8tH9-%3D27DN5xWJ2P9-ROH16e4JUA%40mail.gmail.com

In this patch I recreate those directories to avoid this error. Other solutions
include ignoring the directory-not-existing error or forcing a flush when
redoing those kind of checkpoint xlogs which are added normally in drop
database, etc.

Let's revert or update the code change after the solution is finalized on
upstream.

src/backend/commands/dbcommands.c

@@ -2187,6 +2187,7 @@ dbase_redo(XLogRecPtr beginLoc  __attribute__((unused)), XLogRecPtr lsn  __attri
 		xl_dbase_create_rec *xlrec = (xl_dbase_create_rec *) XLogRecGetData(record);
 		char	   *src_path;
 		char	   *dst_path;
+		char	   *parentdir;
 		struct stat st;
 
 		src_path = GetDatabasePath(xlrec->src_db_id, xlrec->src_tablespace_id);
@@ -2206,6 +2207,30 @@ dbase_redo(XLogRecPtr beginLoc  __attribute__((unused)), XLogRecPtr lsn  __attri
 								dst_path)));
 		}
 
+		/*
+		 * It is possible that the tablespace was later dropped, but we are
+		 * re-redoing database create before that. In that case,
+		 * either src_path or dst_path is probably missing here and needs to
+		 * be created. We create directories here so that copy_dir() won't
+		 * fail, but do not bother to create the symlink under pg_tblspc
+		 * if the tablespace is not global/default.
+		 */
+		if (stat(src_path, &st) != 0 && pg_mkdir_p(src_path, S_IRWXU) != 0)
+		{
+			ereport(WARNING,
+					(errmsg("can not recursively create directory \"%s\"",
+							src_path)));
+		}
+		parentdir = pstrdup(dst_path);
+		get_parent_directory(parentdir);
+		if (stat(parentdir, &st) != 0 && pg_mkdir_p(parentdir, S_IRWXU) != 0)
+		{
+			ereport(WARNING,
+					(errmsg("can not recursively create directory \"%s\"",
+							parentdir)));
+		}
+		pfree(parentdir);
+
 		/*
 		 * Force dirty buffers out to disk, to ensure source database is
 		 * up-to-date for the copy.

src/backend/commands/tablespace.c

@@ -171,8 +171,6 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 				/* Directory creation failed? */
 				if (mkdir(dir, S_IRWXU) < 0)
 				{
-					char	   *parentdir;
-
 					/* Failure other than not exists or not in WAL replay? */
 					if (errno != ENOENT || !isRedo)
 						ereport(ERROR,
@@ -186,30 +184,8 @@ TablespaceCreateDbspace(Oid spcNode, Oid dbNode, bool isRedo)
 					 * than a symlink.
 					 */
 
-					/* create two parents up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (mkdir(parentdir, S_IRWXU) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-							  errmsg("could not create directory \"%s\": %m",
-									 parentdir)));
-					pfree(parentdir);
-
-					/* create one parent up if not exist */
-					parentdir = pstrdup(dir);
-					get_parent_directory(parentdir);
-					/* Can't create parent and it doesn't already exist? */
-					if (mkdir(parentdir, S_IRWXU) < 0 && errno != EEXIST)
-						ereport(ERROR,
-								(errcode_for_file_access(),
-							  errmsg("could not create directory \"%s\": %m",
-									 parentdir)));
-					pfree(parentdir);
-
 					/* Create database directory */
-					if (mkdir(dir, S_IRWXU) < 0)
+					if (pg_mkdir_p(dir, S_IRWXU) < 0)
 						ereport(ERROR,
 								(errcode_for_file_access(),
 							  errmsg("could not create directory \"%s\": %m",