CREATE DATABASE two-phase commit safe (#8078)

When a failed raised by `CREATE DATABASE`, relevant files, directories
should not be leftover.

Add new DB information into `pendingDbDeletes` list, so that promise `CREATE DATABASE` as a 2pc safe command.
Co-authored-by: NAsim R P <apraveen@pivotal.io>

src/backend/commands/dbcommands.c

@@ -641,6 +641,12 @@ createdb(const CreatedbStmt *stmt)
 
 			dstpath = GetDatabasePath(dboid, dsttablespace);
 
+			/*
+			 * Register the database directory to PendingDBDelete link list
+			 * for cleanup in txn abort.
+			 */
+			ScheduleDbDirDelete(dboid, dsttablespace, false);
+
 			/*
 			 * Copy this subdirectory to the new location
 			 *
@@ -648,6 +654,8 @@ createdb(const CreatedbStmt *stmt)
 			 */
 			copydir(srcpath, dstpath, false);
 
+			SIMPLE_FAULT_INJECTOR("create_db_after_file_copy");
+
 			/* Record the filesystem change in XLOG */
 			{
 				xl_dbase_create_rec xlrec;
@@ -666,6 +674,9 @@ createdb(const CreatedbStmt *stmt)
 				(void) XLogInsert(RM_DBASE_ID, XLOG_DBASE_CREATE, rdata);
 			}
 		}
+
+		SIMPLE_FAULT_INJECTOR("after_xlog_create_database");
+
 		heap_endscan(scan);
 		heap_close(rel, AccessShareLock);
 

src/test/regress/expected/.gitignore

@@ -47,4 +47,5 @@ rpt_tpch.out
 session_reset.out
 dropdb_check_shared_buffer_cache.out
 /oid_wraparound.out
-/gp_tablespace_path_too_long.out
\ No newline at end of file
+/gp_tablespace_path_too_long.out
+createdb.out

src/test/regress/greenplum_schedule

@@ -26,6 +26,7 @@ test: instr_in_shmem_setup
 # run separately - because slot counter may influenced by other parallel queries
 test: instr_in_shmem
 
+test: createdb
 test: gp_tablespace gp_aggregates gp_metadata variadic_parameters default_parameters function_extensions spi gp_xml shared_scan update_gp returning_gp resource_queue_with_rule gp_types gp_index
 test: spi_processed64bit
 test: python_processed64bit

src/test/regress/input/createdb.source

+-- start_ignore
+create language plpythonu;
+-- end_ignore
+
+--list all database oid relevant directories in all node(master, segments, mirrors, etc.)
+CREATE OR REPLACE FUNCTION db_dirs(dboid oid) RETURNS setof text
+  STRICT STABLE LANGUAGE plpythonu
+as $$
+import os
+bash_cmd = "find " + os.getcwd() + "/../../ " + "-name %d -type d"
+p = os.popen(bash_cmd % dboid)
+return p.readlines()
+$$;
+
+--this group udf help test case wait mirror catch up
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record_master() RETURNS VOID AS
+'@abs_builddir@/regress.so', 'insert_noop_xlog_record'
+    LANGUAGE C EXECUTE ON MASTER;
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record_all_segments() RETURNS SETOF VOID AS
+'@abs_builddir@/regress.so', 'insert_noop_xlog_record'
+    LANGUAGE C EXECUTE ON ALL SEGMENTS;
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record() RETURNS VOID AS $$
+BEGIN
+    PERFORM insert_noop_xlog_record_master();
+    PERFORM insert_noop_xlog_record_all_segments();
+END;
+$$LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION force_mirrors_to_catch_up() RETURNS VOID AS $$
+BEGIN
+    PERFORM gp_inject_fault2('after_xlog_redo_noop', 'sleep', dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+    PERFORM insert_noop_xlog_record();
+    PERFORM gp_wait_until_triggered_fault2('after_xlog_redo_noop', 1, dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+    PERFORM gp_inject_fault2('after_xlog_redo_noop', 'reset', dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+END;
+$$ LANGUAGE plpgsql;
+
+
+--
+--CASE 0: createdb do well
+--
+create database dowell;
+select force_mirrors_to_catch_up();
+select count(*)=0 as result from
+  (select db_dirs(oid) from pg_database where datname = 'dowell') as foo;
+
+\! psql -d dowell -c "create table test1(a int, b text)"
+\! psql -d dowell -c "insert into test1 values (1, '111'), (2, '222'), (3, '333')"
+\! psql -d dowell -c "select * from test1" 
+
+drop database dowell;
+
+
+
+--
+--CASE 1: error in segment after db file physically created
+--
+--reset status
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+-- inject fault on content0 primary to error out after copying
+-- template db directory
+select gp_inject_fault2('create_db_after_file_copy', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+
+-- should fail
+create database db_with_leftover_files;
+
+-- Wait until replay_location = flush_location.
+select force_mirrors_to_catch_up();
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db_with_leftover_files';
+
+-- cleanup
+set gp_select_invisible=off;
+
+
+
+--
+--CASE 2: error after XLOG_DBASE_CREATE on master
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+select gp_inject_fault2('after_xlog_create_database', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=-1 and role='p';
+-- should fail
+create database db2;
+
+select force_mirrors_to_catch_up();
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db2';
+
+set gp_select_invisible=off;
+
+
+
+--
+--CASE 3: error after XLOG_DBASE_CREATE on segment
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+select gp_inject_fault2('after_xlog_create_database', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+-- should fail
+create database db3;
+
+select force_mirrors_to_catch_up();
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db3';
+
+set gp_select_invisible=off;
+
+
+
+--
+--CASE 4: panic after XLOG_XACT_PREPARE on segment
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+select gp_inject_fault2('end_prepare_two_phase', 'panic', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+select gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+-- should fail
+create database db4;
+
+select force_mirrors_to_catch_up();
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db4';
+
+set gp_select_invisible=off;
+-- start_ignore
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+select force_mirrors_to_catch_up();
+-- end_ignore

src/test/regress/output/createdb.source

+-- start_ignore
+\! gpstop -u;
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Starting gpstop with args: -u
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Gathering information and validating the environment...
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Obtaining Greenplum Master catalog information
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Obtaining Segment details from master...
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Greenplum Version: 'postgres (Greenplum Database) 7.0.0-alpha.0+dev.487.gc712d6d8b0 build dev'
+20190709:16:49:48:083167 gpstop:Weinan-Pivotal-Mac:wwang-[INFO]:-Signalling all postmaster processes to reload
+create language plpythonu;
+-- end_ignore
+--list all database oid relevant directories in all node(master, segments, mirrors, etc.)
+CREATE OR REPLACE FUNCTION db_dirs(dboid oid) RETURNS setof text
+  STRICT STABLE LANGUAGE plpythonu
+as $$
+import os
+bash_cmd = "find " + os.getcwd() + "/../../ " + "-name %d -type d"
+p = os.popen(bash_cmd % dboid)
+return p.readlines()
+$$;
+--this group udf help test case wait mirror catch up
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record_master() RETURNS VOID AS
+'@abs_builddir@/regress.so', 'insert_noop_xlog_record'
+    LANGUAGE C EXECUTE ON MASTER;
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record_all_segments() RETURNS SETOF VOID AS
+'@abs_builddir@/regress.so', 'insert_noop_xlog_record'
+    LANGUAGE C EXECUTE ON ALL SEGMENTS;
+CREATE OR REPLACE FUNCTION insert_noop_xlog_record() RETURNS VOID AS $$
+BEGIN
+    PERFORM insert_noop_xlog_record_master();
+    PERFORM insert_noop_xlog_record_all_segments();
+END;
+$$LANGUAGE plpgsql;
+CREATE OR REPLACE FUNCTION force_mirrors_to_catch_up() RETURNS VOID AS $$
+BEGIN
+    PERFORM gp_inject_fault2('after_xlog_redo_noop', 'sleep', dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+    PERFORM insert_noop_xlog_record();
+    PERFORM gp_wait_until_triggered_fault2('after_xlog_redo_noop', 1, dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+    PERFORM gp_inject_fault2('after_xlog_redo_noop', 'reset', dbid, hostname, port) FROM gp_segment_configuration WHERE role='m';
+END;
+$$ LANGUAGE plpgsql;
+--
+--CASE 0: createdb do well
+--
+create database dowell;
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+select count(*)=0 as result from
+  (select db_dirs(oid) from pg_database where datname = 'dowell') as foo;
+ result 
+--------
+ f
+(1 row)
+
+\! psql -d dowell -c "create table test1(a int, b text)"
+NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table.
+HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
+CREATE TABLE
+\! psql -d dowell -c "insert into test1 values (1, '111'), (2, '222'), (3, '333')"
+INSERT 0 3
+\! psql -d dowell -c "select * from test1" 
+ a |  b  
+---+-----
+ 2 | 222
+ 3 | 333
+ 1 | 111
+(3 rows)
+
+drop database dowell;
+--
+--CASE 1: error in segment after db file physically created
+--
+--reset status
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+ gp_inject_fault2 
+------------------
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+(8 rows)
+
+-- inject fault on content0 primary to error out after copying
+-- template db directory
+select gp_inject_fault2('create_db_after_file_copy', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+ gp_inject_fault2 
+------------------
+ Success:
+(1 row)
+
+-- should fail
+create database db_with_leftover_files;
+ERROR:  fault triggered, fault name:'create_db_after_file_copy' fault type:'error'  (seg0 10.34.49.130:25432 pid=83183)
+-- Wait until replay_location = flush_location.
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db_with_leftover_files';
+ db_dirs 
+---------
+(0 rows)
+
+-- cleanup
+set gp_select_invisible=off;
+--
+--CASE 2: error after XLOG_DBASE_CREATE on master
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+ gp_inject_fault2 
+------------------
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+(8 rows)
+
+select gp_inject_fault2('after_xlog_create_database', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=-1 and role='p';
+ gp_inject_fault2 
+------------------
+ Success:
+(1 row)
+
+-- should fail
+create database db2;
+ERROR:  fault triggered, fault name:'after_xlog_create_database' fault type:'error'
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db2';
+ db_dirs 
+---------
+(0 rows)
+
+set gp_select_invisible=off;
+--
+--CASE 3: error after XLOG_DBASE_CREATE on segment
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+ gp_inject_fault2 
+------------------
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+(8 rows)
+
+select gp_inject_fault2('after_xlog_create_database', 'error', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+ gp_inject_fault2 
+------------------
+ Success:
+(1 row)
+
+-- should fail
+create database db3;
+ERROR:  fault triggered, fault name:'after_xlog_create_database' fault type:'error'  (seg0 10.34.49.130:25432 pid=83183)
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db3';
+ db_dirs 
+---------
+(0 rows)
+
+set gp_select_invisible=off;
+--
+--CASE 4: panic after XLOG_XACT_PREPARE on segment
+--
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+ gp_inject_fault2 
+------------------
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+(8 rows)
+
+select gp_inject_fault2('end_prepare_two_phase', 'panic', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+ gp_inject_fault2 
+------------------
+ Success:
+(1 row)
+
+select gp_inject_fault_infinite2('fts_probe', 'skip', dbid, hostname, port)
+from gp_segment_configuration where content=0 and role='p';
+ gp_inject_fault_infinite2 
+---------------------------
+ Success:
+(1 row)
+
+-- should fail
+create database db4;
+ERROR:  fault triggered, fault name:'end_prepare_two_phase' fault type:'panic'  (seg0 10.34.49.130:25432 pid=83183)
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+-- since this is a failed case, db oid is invisible, however we need it to search 
+-- our tablespace to check if has db files left over.
+set gp_select_invisible=on;
+select db_dirs(oid) from pg_database where datname = 'db4';
+ db_dirs 
+---------
+(0 rows)
+
+set gp_select_invisible=off;
+-- start_ignore
+select gp_inject_fault2('all', 'reset', dbid, hostname, port) from gp_segment_configuration;
+ gp_inject_fault2 
+------------------
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+ Success:
+(8 rows)
+
+select force_mirrors_to_catch_up();
+ force_mirrors_to_catch_up 
+---------------------------
+ 
+(1 row)
+
+-- end_ignore

src/test/regress/serial_schedule

@@ -151,3 +151,4 @@ ignore: largeobject
 test: with
 test: xml
 test: stats
+test: createdb

src/test/regress/sql/.gitignore

@@ -49,4 +49,4 @@ autovacuum-template0.sql
 rpt_tpch.sql
 session_reset.sql
 dropdb_check_shared_buffer_cache.sql
-
+createdb.sql