From 495343e1bbe94d33534fd6161eec8919ccdd55b3 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 23 Mar 2020 16:00:43 +0800 Subject: [PATCH] Revert "Make fault injector configurable (#9532)" (#9795) This reverts commit 7f5c7da134b4881df8c70c49c052f81cdeb5e6c3. --- configure | 35 ----------- configure.in | 9 --- gpcontrib/Makefile | 5 +- gpcontrib/quicklz/quicklz_compression.c | 7 ++- src/Makefile.global.in | 1 - src/backend/access/aocs/aocsam.c | 25 +++++++- src/backend/access/appendonly/appendonlyam.c | 27 ++++++++- src/backend/access/heap/heapam.c | 5 +- src/backend/access/heap/tuptoaster.c | 5 +- src/backend/access/transam/distributedlog.c | 3 +- src/backend/access/transam/xact.c | 18 ++++-- src/backend/access/transam/xlog.c | 6 +- src/backend/cdb/cdbappendonlystoragewrite.c | 6 +- src/backend/cdb/motion/ic_udpifc.c | 6 +- src/backend/commands/copy.c | 4 +- src/backend/commands/vacuum.c | 15 +++-- src/backend/executor/execMain.c | 10 +++- src/backend/executor/spi.c | 9 ++- src/backend/gpopt/gpdbwrappers.cpp | 7 +-- src/backend/postmaster/postmaster.c | 17 +++++- src/backend/utils/sort/tuplesort_mk.c | 9 ++- .../utils/time/test/sharedsnapshot_test.c | 12 ++-- src/include/pg_config.h.in | 3 - src/include/pg_config_manual.h | 5 ++ src/include/utils/faultinjector.h | 8 +-- src/pl/plpython/plpy_spi.c | 5 +- src/test/fsync/Makefile | 9 +-- src/test/gpdb_pitr/Makefile | 2 - src/test/heap_checksum/Makefile | 11 +--- src/test/isolation2/Makefile | 14 +---- .../isolation2_faultinjector_schedule | 57 ------------------ ...isolation2_resgroup_faultinjector_schedule | 6 -- .../isolation2/isolation2_resgroup_schedule | 5 ++ src/test/isolation2/isolation2_schedule | 58 +++++++++++++++++++ src/test/regress/GNUmakefile | 8 +-- src/test/regress/expected/alter_extension.out | 13 +++-- .../regress/expected/psql_gp_commands.out | 33 ++++++----- .../regress/greenplum_faultinjector_schedule | 26 --------- src/test/regress/greenplum_schedule | 45 +++++++++++++- .../regress/scan_flaky_fault_injectors.sh | 17 +----- src/test/regress/serial_schedule | 1 + src/test/regress/sql/alter_extension.sql | 12 ++-- src/test/regress/sql/psql_gp_commands.sql | 8 ++- src/test/walrep/Makefile | 6 +- 44 files changed, 315 insertions(+), 278 deletions(-) delete mode 100644 src/test/isolation2/isolation2_faultinjector_schedule delete mode 100644 src/test/isolation2/isolation2_resgroup_faultinjector_schedule delete mode 100644 src/test/regress/greenplum_faultinjector_schedule diff --git a/configure b/configure index c1f852e6d6..ce91e6401e 100755 --- a/configure +++ b/configure @@ -744,7 +744,6 @@ HAVE_CXX11 enable_gpcloud enable_mapreduce enable_orca -enable_faultinjector autodepend TAS GCC @@ -859,7 +858,6 @@ with_wal_segsize with_CC enable_depend enable_cassert -enable_faultinjector enable_orca enable_mapreduce enable_gpcloud @@ -1543,7 +1541,6 @@ Optional Features: --enable-tap-tests enable TAP tests (requires Perl and IPC::Run) --enable-depend turn on automatic dependency tracking --enable-cassert enable assertion checks (for debugging) - --enable-faultinjector enable fault injector --disable-orca disable ORCA optimizer --enable-mapreduce enable Greenplum Mapreduce support --disable-gpcloud disable gpcloud support @@ -6459,38 +6456,6 @@ fi -# -# Enable fault injector -# - - -# Check whether --enable-faultinjector was given. -if test "${enable_faultinjector+set}" = set; then : - enableval=$enable_faultinjector; - case $enableval in - yes) - -$as_echo "#define FAULT_INJECTOR 1" >>confdefs.h - - ;; - no) - : - ;; - *) - as_fn_error $? "no argument expected for --enable-faultinjector option" "$LINENO" 5 - ;; - esac - -else - enable_faultinjector=no - -fi - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with FAULT_INJECTOR... $enable_faultinjector" >&5 -$as_echo "checking whether to build with FAULT_INJECTOR... $enable_faultinjector" >&6; } - - # # Enable GPORCA optimizer # diff --git a/configure.in b/configure.in index ad8ca97ed1..c84e843296 100644 --- a/configure.in +++ b/configure.in @@ -799,15 +799,6 @@ PGAC_ARG_BOOL(enable, cassert, no, [enable assertion checks (for debugging)], [AC_DEFINE([USE_ASSERT_CHECKING], 1, [Define to 1 to build with assertion checks. (--enable-cassert)])]) -# -# Enable fault injector -# -PGAC_ARG_BOOL(enable, faultinjector, no, [enable fault injector], - [AC_DEFINE([FAULT_INJECTOR], 1, - [Define to 1 to build with fault injector. (--enable-faultinjector)])]) -AC_MSG_RESULT([checking whether to build with FAULT_INJECTOR... $enable_faultinjector]) -AC_SUBST(enable_faultinjector) - # # Enable GPORCA optimizer # diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index aa49041f12..64a280f537 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -17,6 +17,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_distribution_policy \ gp_internal_tools \ gp_debug_numsegments \ + gp_inject_fault \ gp_replica_check else recurse_targets = gp_sparse_vector \ @@ -24,10 +25,6 @@ else gp_internal_tools endif -ifeq "$(enable_faultinjector)" "yes" - recurse_targets += gp_inject_fault -endif - ifeq "$(with_zstd)" "yes" recurse_targets += zstd endif diff --git a/gpcontrib/quicklz/quicklz_compression.c b/gpcontrib/quicklz/quicklz_compression.c index 2861192866..f2f8144980 100644 --- a/gpcontrib/quicklz/quicklz_compression.c +++ b/gpcontrib/quicklz/quicklz_compression.c @@ -98,7 +98,12 @@ quicklz_decompressor(int level, const char *source, void *destination, Datum quicklz_constructor(PG_FUNCTION_ARGS) { - SIMPLE_FAULT_INJECTOR("malloc_failure"); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet("malloc_failure", + DDLNotSpecified, + "", // databaseName + ""); // tableName +#endif /* PG_GETARG_POINTER(0) is TupleDesc that is currently unused. * It is passed as NULL */ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 6105998c88..9f07053217 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -209,7 +209,6 @@ enable_rpath = @enable_rpath@ enable_debug = @enable_debug@ enable_dtrace = @enable_dtrace@ enable_coverage = @enable_coverage@ -enable_faultinjector = @enable_faultinjector@ enable_tap_tests = @enable_tap_tests@ enable_thread_safety = @enable_thread_safety@ enable_strong_random = @enable_strong_random@ diff --git a/src/backend/access/aocs/aocsam.c b/src/backend/access/aocs/aocsam.c index e42660673d..15afa4081a 100644 --- a/src/backend/access/aocs/aocsam.c +++ b/src/backend/access/aocs/aocsam.c @@ -885,7 +885,13 @@ aocs_insert_values(AOCSInsertDesc idesc, Datum *d, bool *null, AOTupleId *aoTupl (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("append-only column-oriented tables do not support rows with OIDs"))); - FAULT_INJECTOR_TABLE("appendonly_insert", RelationGetRelationName(idesc->aoi_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_insert", + DDLNotSpecified, + "", /* databaseName */ + RelationGetRelationName(idesc->aoi_rel)); /* tableName */ +#endif /* As usual, at this moment, we assume one col per vp */ for (i = 0; i < RelationGetNumberOfAttributes(rel); ++i) @@ -1585,7 +1591,14 @@ aocs_update(AOCSUpdateDesc desc, TupleTableSlot *slot, Assert(oldTupleId); Assert(newTupleId); - FAULT_INJECTOR_TABLE("appendonly_update", RelationGetRelationName(desc->insertDesc->aoi_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_update", + DDLNotSpecified, + "", //databaseName + RelationGetRelationName(desc->insertDesc->aoi_rel)); + /* tableName */ +#endif result = AppendOnlyVisimapDelete_Hide(&desc->visiMapDelete, oldTupleId); if (result != HeapTupleMayBeUpdated) @@ -1679,7 +1692,13 @@ aocs_delete(AOCSDeleteDesc aoDeleteDesc, NameStr(aoDeleteDesc->aod_rel->rd_rel->relname), AOTupleIdToString(aoTupleId)); - FAULT_INJECTOR_TABLE("appendonly_delete", RelationGetRelationName(aoDeleteDesc->aod_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_delete", + DDLNotSpecified, + "", /* databaseName */ + RelationGetRelationName(aoDeleteDesc->aod_rel)); /* tableName */ +#endif return AppendOnlyVisimapDelete_Hide(&aoDeleteDesc->visiMapDelete, aoTupleId); } diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c index c5ba1de009..71b15246d6 100755 --- a/src/backend/access/appendonly/appendonlyam.c +++ b/src/backend/access/appendonly/appendonlyam.c @@ -2457,7 +2457,14 @@ appendonly_delete(AppendOnlyDeleteDesc aoDeleteDesc, NameStr(aoDeleteDesc->aod_rel->rd_rel->relname), AOTupleIdToString(aoTupleId)); - FAULT_INJECTOR_TABLE("appendonly_delete", RelationGetRelationName(aoDeleteDesc->aod_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_delete", + DDLNotSpecified, + "", //databaseName + RelationGetRelationName(aoDeleteDesc->aod_rel)); + /* tableName */ +#endif return AppendOnlyVisimapDelete_Hide(&aoDeleteDesc->visiMapDelete, aoTupleId); } @@ -2521,7 +2528,14 @@ appendonly_update(AppendOnlyUpdateDesc aoUpdateDesc, Assert(aoUpdateDesc); Assert(aoTupleId); - FAULT_INJECTOR_TABLE("appendonly_update", RelationGetRelationName(aoUpdateDesc->aoInsertDesc->aoi_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_update", + DDLNotSpecified, + "", //databaseName + RelationGetRelationName(aoUpdateDesc->aoInsertDesc->aoi_rel)); + /* tableName */ +#endif result = AppendOnlyVisimapDelete_Hide(&aoUpdateDesc->visiMapDelete, aoTupleId); if (result != HeapTupleMayBeUpdated) @@ -2788,7 +2802,14 @@ appendonly_insert(AppendOnlyInsertDesc aoInsertDesc, Assert(aoInsertDesc->usableBlockSize > 0 && aoInsertDesc->tempSpaceLen > 0); Assert(aoInsertDesc->toast_tuple_threshold > 0 && aoInsertDesc->toast_tuple_target > 0); - FAULT_INJECTOR_TABLE("appendonly_insert", RelationGetRelationName(aoInsertDesc->aoi_rel)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet( + "appendonly_insert", + DDLNotSpecified, + "", //databaseName + RelationGetRelationName(aoInsertDesc->aoi_rel)); + /* tableName */ +#endif Assert(RelationIsAoRows(relation)); diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 08e096606b..71c5329639 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -2683,7 +2683,10 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, needwal = !(options & HEAP_INSERT_SKIP_WAL) && RelationNeedsWAL(relation); gp_expand_protect_catalog_changes(relation); - FAULT_INJECTOR_TABLE("heap_insert", RelationGetRelationName(relation)); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet("heap_insert", DDLNotSpecified, "", + RelationGetRelationName(relation)); +#endif /* * Fill in tuple header fields, assign an OID, and toast the tuple if diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index b325c272d9..4c0ed0ccdd 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -1824,7 +1824,10 @@ toast_save_datum(Relation rel, Datum value, * modified (here, we decrease it by one). The result must still fit into * TOAST_MAX_CHUNK_SIZE so that it doesn't overflow our chunk_data struct. */ - if (SIMPLE_FAULT_INJECTOR("decrease_toast_max_chunk_size") != FaultInjectorTypeNotSpecified) + if (FaultInjector_InjectFaultIfSet("decrease_toast_max_chunk_size", + DDLNotSpecified, + "", /* databaseName */ + "" /* tableName */) != FaultInjectorTypeNotSpecified) { max_chunk_size--; } diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index 13285c2340..8e19366daa 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -189,7 +189,8 @@ DistributedLog_AdvanceOldestXmin(TransactionId oldestLocalXmin, if (MyProcPort) dbname = MyProcPort->database_name; - FAULT_INJECTOR_DATABASE("distributedlog_advance_oldest_xmin", dbname ? dbname : ""); + FaultInjector_InjectFaultIfSet("distributedlog_advance_oldest_xmin", DDLNotSpecified, + dbname?dbname: "", ""); #endif LWLockAcquire(DistributedLogTruncateLock, LW_SHARED); diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index af5c82f503..d5ffb727a4 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -1604,7 +1604,10 @@ RecordTransactionCommit(void) if (isDtxPrepared == 0 && CurrentTransactionState->blockState == TBLOCK_END) { - SIMPLE_FAULT_INJECTOR("local_tm_record_transaction_commit"); + FaultInjector_InjectFaultIfSet("local_tm_record_transaction_commit", + DDLNotSpecified, + "", // databaseName + ""); // tableName } #endif @@ -1658,7 +1661,10 @@ RecordTransactionCommit(void) #ifdef FAULT_INJECTOR if (isDtxPrepared) { - SIMPLE_FAULT_INJECTOR("dtm_xlog_distributed_commit"); + FaultInjector_InjectFaultIfSet("dtm_xlog_distributed_commit", + DDLNotSpecified, + "", // databaseName + ""); // tableName } #endif @@ -2707,9 +2713,13 @@ CommitTransaction(void) #ifdef FAULT_INJECTOR if (isPreparedDtxTransaction()) { - SIMPLE_FAULT_INJECTOR("transaction_abort_after_distributed_prepare"); + FaultInjector_InjectFaultIfSet( + "transaction_abort_after_distributed_prepared", + DDLNotSpecified, + "", // databaseName + ""); // tableName } -#endif +#endif if (Debug_abort_after_distributed_prepared && isPreparedDtxTransaction()) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index d6ce27c785..109e338ec4 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8652,7 +8652,11 @@ CreateCheckPoint(int flags) } #ifdef FAULT_INJECTOR - if (SIMPLE_FAULT_INJECTOR("checkpoint") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet( + "checkpoint", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) return; // skip checkpoint #endif diff --git a/src/backend/cdb/cdbappendonlystoragewrite.c b/src/backend/cdb/cdbappendonlystoragewrite.c index 134550325c..485428a8d8 100755 --- a/src/backend/cdb/cdbappendonlystoragewrite.c +++ b/src/backend/cdb/cdbappendonlystoragewrite.c @@ -1169,7 +1169,11 @@ AppendOnlyStorageWrite_CompressAppend(AppendOnlyStorageWrite *storageWrite, #ifdef FAULT_INJECTOR /* Simulate that compression is not possible if the fault is set. */ - if (FAULT_INJECTOR_TABLE("appendonly_skip_compression", storageWrite->relationName) == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet( + "appendonly_skip_compression", + DDLNotSpecified, + "", + storageWrite->relationName) == FaultInjectorTypeSkip) { *compressedLen = sourceLen + 1; } diff --git a/src/backend/cdb/motion/ic_udpifc.c b/src/backend/cdb/motion/ic_udpifc.c index 6d4ebefc7d..6ee431a5a4 100644 --- a/src/backend/cdb/motion/ic_udpifc.c +++ b/src/backend/cdb/motion/ic_udpifc.c @@ -5703,7 +5703,11 @@ doSendStopMessageUDPIFC(ChunkTransportState *transportStates, int16 motNodeID) */ #ifdef FAULT_INJECTOR - if (SIMPLE_FAULT_INJECTOR("interconnect_stop_ack_is_lost") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet( + "interconnect_stop_ack_is_lost", + DDLNotSpecified, + "" /* databaseName */ , + "" /* tableName */ ) == FaultInjectorTypeSkip) { continue; } diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index a47464b76c..c5928d11f7 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2667,7 +2667,9 @@ DoCopyTo(CopyState cstate) bool fe_copy = (pipe && whereToSendOutput == DestRemote); uint64 processed; - SIMPLE_FAULT_INJECTOR("DoCopyToFail"); +#ifdef FAULT_INJECTOR + FaultInjector_InjectFaultIfSet("DoCopyToFail", DDLNotSpecified, "", ""); +#endif PG_TRY(); { diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index c42b6bebb8..dd5a179a5e 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -1827,10 +1827,17 @@ vacuum_rel(Oid relid, RangeVar *relation, int options, VacuumParams *params, if (ao_vacuum_phase == VACOPT_AO_POST_CLEANUP_PHASE) { - FAULT_INJECTOR_TABLE("compaction_before_cleanup_phase", - RelationGetRelationName(onerel)); - FAULT_INJECTOR_TABLE("compaction_before_segmentfile_drop", - RelationGetRelationName(onerel)); + FaultInjector_InjectFaultIfSet( + "compaction_before_cleanup_phase", + DDLNotSpecified, + "", // databaseName + RelationGetRelationName(onerel)); // tableName + + FaultInjector_InjectFaultIfSet( + "compaction_before_segmentfile_drop", + DDLNotSpecified, + "", // databaseName + RelationGetRelationName(onerel)); // tableName } /* diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index 936c82dcfb..e82f2dba29 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -904,7 +904,10 @@ standard_ExecutorRun(QueryDesc *queryDesc, if (estate->es_processed >= 10000 && estate->es_processed <= 1000000) //if (estate->es_processed >= 10000) { - if (SIMPLE_FAULT_INJECTOR("executor_run_high_processed") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet("executor_run_high_processed", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { /* * For testing purposes, pretend that we have already processed @@ -2890,7 +2893,10 @@ ExecutePlan(EState *estate, */ if (estate->es_processed >= 10000 && estate->es_processed <= 1000000) { - if (SIMPLE_FAULT_INJECTOR("executor_run_high_processed") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet("executor_run_high_processed", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { /* * For testing purposes, pretend that we have already processed diff --git a/src/backend/executor/spi.c b/src/backend/executor/spi.c index 3335b151c7..e4e839d247 100644 --- a/src/backend/executor/spi.c +++ b/src/backend/executor/spi.c @@ -2650,10 +2650,17 @@ _SPI_pquery(QueryDesc *queryDesc, bool fire_triggers, uint64 tcount) /* * only check number tuples if the SPI 64 bit test is NOT running */ - if (!SIMPLE_FAULT_INJECTOR("executor_run_high_processed")) + if (!FaultInjector_InjectFaultIfSet("executor_run_high_processed", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */)) + { #endif /* FAULT_INJECTOR */ if (_SPI_checktuples()) elog(ERROR, "consistency check on SPI tuple count failed"); +#ifdef FAULT_INJECTOR + } +#endif /* FAULT_INJECTOR */ } /* MPP-14001: Running auto_stats */ diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index 20316bfa10..159b84cc15 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -3002,17 +3002,12 @@ gpdb::InjectFaultInOptTasks const char *fault_name ) { -#ifdef FAULT_INJECTOR GP_WRAP_START; { - return SIMPLE_FAULT_INJECTOR(fault_name); + return FaultInjector_InjectFaultIfSet(fault_name, DDLNotSpecified, "", ""); } GP_WRAP_END; return FaultInjectorTypeNotSpecified; -#else - GPOS_RAISE(gpdxl::ExmaGPDB, gpdxl::ExmiGPDBError, - GPOS_WSZ_LIT("InjectFaultInOptTasks can only be called when fault injector is enabled")); -#endif } gpos::ULONG diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 347224f85e..d99ac9bbf1 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2368,14 +2368,22 @@ retry1: am_ftshandler = true; #ifdef FAULT_INJECTOR - if (SIMPLE_FAULT_INJECTOR("fts_conn_startup_packet") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet( + "fts_conn_startup_packet", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { /* * If this fault is set to skip, report recovery is * hung. Without this fault recovery is reported as * progressing. */ - if (SIMPLE_FAULT_INJECTOR("fts_recovery_in_progress") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet( + "fts_recovery_in_progress", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { recptr = last_xlog_replay_location(); } @@ -2581,7 +2589,10 @@ retry1: #ifdef FAULT_INJECTOR if (!am_ftshandler && !IsFaultHandler && !am_walsender && - FAULT_INJECTOR_DATABASE("process_startup_packet", port->database_name) == FaultInjectorTypeSkip) + FaultInjector_InjectFaultIfSet("process_startup_packet", + DDLNotSpecified, + port->database_name /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { ereport(FATAL, (errcode(ERRCODE_CANNOT_CONNECT_NOW), diff --git a/src/backend/utils/sort/tuplesort_mk.c b/src/backend/utils/sort/tuplesort_mk.c index 9dda8b9a9b..511bc88ae1 100644 --- a/src/backend/utils/sort/tuplesort_mk.c +++ b/src/backend/utils/sort/tuplesort_mk.c @@ -2028,8 +2028,13 @@ mergeruns(Tuplesortstate_mk *state) * handled within the fault injector itself. */ HOLD_INTERRUPTS(); - SIMPLE_FAULT_INJECTOR("execsort_mksort_mergeruns"); - RESUME_INTERRUPTS(); + FaultInjector_InjectFaultIfSet( + "execsort_mksort_mergeruns", + DDLNotSpecified, + "", //databaseName + ""); + //tableName + RESUME_INTERRUPTS(); #endif if (QueryFinishPending) diff --git a/src/backend/utils/time/test/sharedsnapshot_test.c b/src/backend/utils/time/test/sharedsnapshot_test.c index 9ac3e60198..c5de2e58ff 100644 --- a/src/backend/utils/time/test/sharedsnapshot_test.c +++ b/src/backend/utils/time/test/sharedsnapshot_test.c @@ -58,13 +58,11 @@ test_write_read_shared_snapshot_for_cursor(void **state) expect_any(LWLockAcquire, mode); will_be_called(LWLockAcquire); -#ifdef FAULT_INJECTOR - expect_any_count(FaultInjector_InjectFaultIfSet, faultName, 11); - expect_any_count(FaultInjector_InjectFaultIfSet, ddlStatement, 11); - expect_any_count(FaultInjector_InjectFaultIfSet, databaseName, 11); - expect_any_count(FaultInjector_InjectFaultIfSet, tableName, 11); - will_be_called_count(FaultInjector_InjectFaultIfSet, 11); -#endif + expect_any_count(FaultInjector_InjectFaultIfSet, faultName, 9); + expect_any_count(FaultInjector_InjectFaultIfSet, ddlStatement, 9); + expect_any_count(FaultInjector_InjectFaultIfSet, databaseName, 9); + expect_any_count(FaultInjector_InjectFaultIfSet, tableName, 9); + will_be_called_count(FaultInjector_InjectFaultIfSet, 9); expect_any(LWLockRelease, lock); will_be_called(LWLockRelease); diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index 859d886276..41f1b68d53 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -58,9 +58,6 @@ (--enable-thread-safety) */ #undef ENABLE_THREAD_SAFETY -/* Define to 1 to build with fault injector. (--enable-faultinjector) */ -#undef FAULT_INJECTOR - /* Define to nothing if C supports flexible array members, and to 1 if it does not. That way, with a declaration like `struct s { int n; double d[FLEXIBLE_ARRAY_MEMBER]; };', the struct hack can be used with pre-C99 diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index cd1e325815..6bf4a75660 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -292,6 +292,11 @@ */ /* #define WAL_DEBUG */ +/* + * Enable injecting faults. + */ +#define FAULT_INJECTOR 1 + /* * Enable tracing of resource consumption during sort operations; * see also the trace_sort GUC var. For 8.1 this is enabled by default. diff --git a/src/include/utils/faultinjector.h b/src/include/utils/faultinjector.h index e1919782b0..df5e93f141 100644 --- a/src/include/utils/faultinjector.h +++ b/src/include/utils/faultinjector.h @@ -9,7 +9,7 @@ #ifndef FAULTINJECTOR_H #define FAULTINJECTOR_H -#include "pg_config.h" +#include "pg_config_manual.h" #define FAULTINJECTOR_MAX_SLOTS 16 @@ -105,15 +105,9 @@ extern bool am_faulthandler; #define IsFaultHandler am_faulthandler #define SIMPLE_FAULT_INJECTOR(FaultName) \ FaultInjector_InjectFaultIfSet(FaultName, DDLNotSpecified, "", "") -#define FAULT_INJECTOR_DATABASE(FaultName, DatabaseName) \ - FaultInjector_InjectFaultIfSet(FaultName, DDLNotSpecified, DatabaseName, "") -#define FAULT_INJECTOR_TABLE(FaultName, TableName) \ - FaultInjector_InjectFaultIfSet(FaultName, DDLNotSpecified, "", TableName) #else #define IsFaultHandler false #define SIMPLE_FAULT_INJECTOR(FaultName) -#define FAULT_INJECTOR_DATABASE(FaultName, DatabaseName) -#define FAULT_INJECTOR_TABLE(FaultName, TableName) #endif #endif /* FAULTINJECTOR_H */ diff --git a/src/pl/plpython/plpy_spi.c b/src/pl/plpython/plpy_spi.c index 7c3bcddca3..7364d61b95 100644 --- a/src/pl/plpython/plpy_spi.c +++ b/src/pl/plpython/plpy_spi.c @@ -414,7 +414,10 @@ PLy_spi_execute_fetch_result(SPITupleTable *tuptable, uint64 rows, int status) #ifdef FAULT_INJECTOR if (rows >= 10000 && rows <= 1000000) { - if (SIMPLE_FAULT_INJECTOR("executor_run_high_processed") == FaultInjectorTypeSkip) + if (FaultInjector_InjectFaultIfSet("executor_run_high_processed", + DDLNotSpecified, + "" /* databaseName */, + "" /* tableName */) == FaultInjectorTypeSkip) { /* * For testing purposes, pretend that we have already processed diff --git a/src/test/fsync/Makefile b/src/test/fsync/Makefile index b6957e815d..f98b1ec10c 100644 --- a/src/test/fsync/Makefile +++ b/src/test/fsync/Makefile @@ -1,17 +1,14 @@ MODULES=fsync_helper PG_CONFIG=pg_config +REGRESS = setup bgwriter_checkpoint +REGRESS_OPTS = --load-extension=gp_inject_fault + subdir = src/test/fsync/ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -REGRESS = setup -ifeq ($(enable_faultinjector),yes) -REGRESS += bgwriter_checkpoint -REGRESS_OPTS = --load-extension=gp_inject_fault -endif - NO_PGXS = 1 include $(top_srcdir)/src/makefiles/pgxs.mk diff --git a/src/test/gpdb_pitr/Makefile b/src/test/gpdb_pitr/Makefile index 9137e80319..3f75ef763c 100644 --- a/src/test/gpdb_pitr/Makefile +++ b/src/test/gpdb_pitr/Makefile @@ -12,9 +12,7 @@ install: sql_isolation_testcase.py $(MAKE) -C $(top_builddir)/src/test/isolation2 install installcheck: install -ifeq ($(enable_faultinjector),yes) ./test_gpdb_pitr.sh -endif clean: ./test_gpdb_pitr_cleanup.sh diff --git a/src/test/heap_checksum/Makefile b/src/test/heap_checksum/Makefile index c28382b4e1..e340dcd41a 100644 --- a/src/test/heap_checksum/Makefile +++ b/src/test/heap_checksum/Makefile @@ -1,19 +1,14 @@ MODULES=heap_checksum_helper PG_CONFIG=pg_config +REGRESS = setup heap_checksum_corruption +REGRESS_OPTS = --init-file=../regress/init_file --load-extension=gp_inject_fault + subdir = src/test/heap_checksum/ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -REGRESS = setup -REGRESS_OPTS = --init-file=../regress/init_file - -ifeq ($(enable_faultinjector),yes) -REGRESS += heap_checksum_corruption -REGRESS_OPTS += --load-extension=gp_inject_fault -endif - NO_PGXS = 1 include $(top_srcdir)/src/makefiles/pgxs.mk diff --git a/src/test/isolation2/Makefile b/src/test/isolation2/Makefile index d0f29b702b..8c3aada3f4 100644 --- a/src/test/isolation2/Makefile +++ b/src/test/isolation2/Makefile @@ -14,10 +14,6 @@ ifeq ($(PORTNAME), win32) LDLIBS += -lws2_32 endif -ifeq ($(enable_faultinjector),yes) -EXTRA_REGRESS_OPTS += --load-extension=gp_inject_fault -endif - override CPPFLAGS := -I$(srcdir) -I$(libpq_srcdir) -I$(srcdir)/../regress $(CPPFLAGS) override LDLIBS := $(libpq_pgport) $(LDLIBS) @@ -68,13 +64,7 @@ clean distclean: install: all gpdiff.pl gpstringsubs.pl installcheck: install - ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --bindir='$(bindir)' --inputdir=$(srcdir) --schedule=$(srcdir)/isolation2_schedule -ifeq ($(enable_faultinjector),yes) - ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --bindir='$(bindir)' --inputdir=$(srcdir) --schedule=$(srcdir)/isolation2_faultinjector_schedule -endif + ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_isolation2 --bindir='$(bindir)' --inputdir=$(srcdir) --load-extension=gp_inject_fault --schedule=$(srcdir)/isolation2_schedule installcheck-resgroup: install - ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --bindir='$(bindir)' --inputdir=$(srcdir) --dbname=isolation2resgrouptest --schedule=$(srcdir)/isolation2_resgroup_schedule -ifeq ($(enable_faultinjector),yes) - ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --bindir='$(bindir)' --inputdir=$(srcdir) --dbname=isolation2resgrouptest --schedule=$(srcdir)/isolation2_resgroup_faultinjector_schedule -endif + ./pg_isolation2_regress $(EXTRA_REGRESS_OPTS) --init-file=$(top_builddir)/src/test/regress/init_file --init-file=./init_file_resgroup --bindir='$(bindir)' --inputdir=$(srcdir) --dbname=isolation2resgrouptest --load-extension=gp_inject_fault --schedule=$(srcdir)/isolation2_resgroup_schedule diff --git a/src/test/isolation2/isolation2_faultinjector_schedule b/src/test/isolation2/isolation2_faultinjector_schedule deleted file mode 100644 index c906bcc944..0000000000 --- a/src/test/isolation2/isolation2_faultinjector_schedule +++ /dev/null @@ -1,57 +0,0 @@ -# This schedule file only contains tests that depend on fault injector - -# Tests for crash recovery -test: uao_crash_compaction_column -test: uao_crash_compaction_row -test: crash_recovery -test: crash_recovery_redundant_dtx -test: crash_recovery_dtm -test: unlogged_heap_tables -test: unlogged_appendonly_tables -test: udf_exception_blocks_panic_scenarios -test: ao_same_trans_truncate_crash -# Tests for packcore, will use the coredumps generated in crash_recovery_dtm, -# so must be scheduled after that one. -test: packcore - -# Tests on global deadlock detector -test: gdd/avoid-qd-deadlock -test: gdd/planner_insert_while_vacuum_drop - -# The following test injects a fault at a generic location -# (StartTransaction). The fault can be easily triggered by a -# concurrent test, so run the test by itself. -test: deadlock_under_entry_db_singleton - -test: concurrent_update -test: pg_terminate_backend -test: terminate_in_gang_creation -test: reindex -test: reindex_gpfastsequence -test: commit_transaction_block_checkpoint -test: distributed_snapshot -test: gp_collation -test: external_table -# This test validates that for AO we delay fsync to checkpointer on mirror. -test: fsync_ao -test: concurrent_index_creation_should_not_deadlock -# Tests on Append-Optimized tables -test: uao/vacuum_cleanup_row -test: uao/insert_should_not_use_awaiting_drop_row -test: segwalrep/master_xlog_switch -test: uao/vacuum_cleanup_column -test: uao/insert_should_not_use_awaiting_drop_column -test: segwalrep/die_commit_pending_replication - -# Tests for FTS -test: fts_errors -test: segwalrep/commit_blocking -test: segwalrep/fts_unblock_primary -test: segwalrep/mirror_promotion -test: segwalrep/cancel_commit_pending_replication -test: segwalrep/twophase_tolerance_with_mirror_promotion -test: segwalrep/failover_with_many_records -test: segwalrep/dtm_recovery_on_standby -test: segwalrep/commit_blocking_on_standby -test: segwalrep/dtx_recovery_wait_lsn -test: fts_manual_probe diff --git a/src/test/isolation2/isolation2_resgroup_faultinjector_schedule b/src/test/isolation2/isolation2_resgroup_faultinjector_schedule deleted file mode 100644 index e46d31609f..0000000000 --- a/src/test/isolation2/isolation2_resgroup_faultinjector_schedule +++ /dev/null @@ -1,6 +0,0 @@ -# This schedule file only contains tests that depend on fault injector - -test: resgroup/resgroup_assign_slot_fail -test: resgroup/resgroup_unassign_entrydb -test: resgroup/resgroup_cpuset - diff --git a/src/test/isolation2/isolation2_resgroup_schedule b/src/test/isolation2/isolation2_resgroup_schedule index 2e11d1ab5e..68e4db9fbc 100644 --- a/src/test/isolation2/isolation2_resgroup_schedule +++ b/src/test/isolation2/isolation2_resgroup_schedule @@ -7,6 +7,10 @@ test: resgroup/resgroup_syntax test: resgroup/resgroup_transaction test: resgroup/resgroup_name_convention +# fault injection tests +test: resgroup/resgroup_assign_slot_fail +test: resgroup/resgroup_unassign_entrydb + # functions test: resgroup/resgroup_concurrency test: resgroup/resgroup_bypass @@ -18,6 +22,7 @@ test: resgroup/resgroup_memory_runaway test: resgroup/resgroup_alter_memory test: resgroup/resgroup_cpu_rate_limit test: resgroup/resgroup_alter_memory_spill_ratio +test: resgroup/resgroup_cpuset test: resgroup/resgroup_cpuset_empty_default test: resgroup/resgroup_set_memory_spill_ratio test: resgroup/resgroup_unlimit_memory_spill_ratio diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 37482edf7e..4081f8959e 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -7,6 +7,21 @@ test: dml_on_root_locks_all_parts test: select_dropped_table test: update_hash_col_utilitymode execute_on_utilitymode +# Tests for crash recovery +test: uao_crash_compaction_column +test: uao_crash_compaction_row +test: crash_recovery +test: crash_recovery_redundant_dtx +test: crash_recovery_dtm +test: unlogged_heap_tables +test: unlogged_appendonly_tables +test: udf_exception_blocks_panic_scenarios +test: ao_same_trans_truncate_crash + +# Tests for packcore, will use the coredumps generated in crash_recovery_dtm, +# so must be scheduled after that one. +test: packcore + # Tests on global deadlock detector test: gdd/prepare test: gdd/dist-deadlock-01 gdd/dist-deadlock-04 gdd/dist-deadlock-05 gdd/dist-deadlock-06 gdd/dist-deadlock-07 gdd/dist-deadlock-102 gdd/dist-deadlock-103 gdd/dist-deadlock-104 gdd/dist-deadlock-106 gdd/dist-deadlock-upsert gdd/non-lock-105 @@ -14,6 +29,7 @@ test: gdd/dist-deadlock-01 gdd/dist-deadlock-04 gdd/dist-deadlock-05 gdd/dist-de ignore: gdd/non-lock-107 # keep this in a separate group test: gdd/extended_protocol_test +test: gdd/avoid-qd-deadlock test: gdd/delete-deadlock-root-leaf-concurrent-op test: gdd/update-deadlock-root-leaf-concurrent-op test: gdd/dml_locks_only_targeted_table_in_query @@ -26,24 +42,47 @@ test: gdd/end test: modify_table_data_corrupt +# The following test injects a fault at a generic location +# (StartTransaction). The fault can be easily triggered by a +# concurrent test, so run the test by itself. +test: deadlock_under_entry_db_singleton + # this case creates table & index in utility mode, which may cause oid # conflict when running in parallel with other cases. test: misc test: starve_case pg_views_concurrent_drop alter_blocks_for_update_and_viceversa drop_rename reader_waits_for_lock resource_queue +# below test(s) inject faults so each of them need to be in a separate group +test: pg_terminate_backend + +# this case contains fault injection, must be put in a separate test group +test: terminate_in_gang_creation + # below case will cause failures on catalog changes, # please keep it in a separate test group test: gpexpand_catalog_lock +test: reindex +test: reindex_gpfastsequence +test: commit_transaction_block_checkpoint test: instr_in_shmem_setup test: instr_in_shmem_terminate test: vacuum_recently_dead_tuple_due_to_distributed_snapshot test: distributedlog-bug test: invalidated_toast_index +test: distributed_snapshot +test: gp_collation test: ao_upgrade +# below test utilizes fault injectors so it needs to be in a group by itself +test: external_table + +# This test validates that for AO we delay fsync to checkpointer on mirror. +test: fsync_ao + # Tests on Append-Optimized tables (row-oriented). +test: concurrent_index_creation_should_not_deadlock test: uao/alter_while_vacuum_row uao/alter_while_vacuum2_row test: uao/compaction_full_stats_row test: uao/compaction_utility_row @@ -87,7 +126,10 @@ test: uao/vacuum_self_serializable3_row test: uao/vacuum_self_function_row test: uao/vacuum_while_insert_row test: uao/vacuum_while_vacuum_row +test: uao/vacuum_cleanup_row test: reorganize_after_ao_vacuum_skip_drop truncate_after_ao_vacuum_skip_drop mark_all_aoseg_await_drop +# below test(s) inject faults so each of them need to be in a separate group +test: segwalrep/master_xlog_switch # Tests on Append-Optimized tables (column-oriented). test: uao/alter_while_vacuum_column uao/alter_while_vacuum2_column @@ -133,11 +175,26 @@ test: uao/vacuum_self_serializable3_column test: uao/vacuum_self_function_column test: uao/vacuum_while_insert_column test: uao/vacuum_while_vacuum_column +test: uao/vacuum_cleanup_column test: add_column_after_vacuum_skip_drop_column test: vacuum_after_vacuum_skip_drop_column +test: segwalrep/die_commit_pending_replication + +# Tests for FTS +test: fts_errors +test: segwalrep/commit_blocking +test: segwalrep/fts_unblock_primary +test: segwalrep/mirror_promotion +test: segwalrep/cancel_commit_pending_replication +test: segwalrep/twophase_tolerance_with_mirror_promotion +test: segwalrep/failover_with_many_records +test: segwalrep/dtm_recovery_on_standby +test: segwalrep/commit_blocking_on_standby +test: segwalrep/dtx_recovery_wait_lsn test: pg_basebackup test: pg_basebackup_with_tablespaces +test: fts_manual_probe # Reindex tests test: reindex/abort_reindex @@ -161,6 +218,7 @@ test: reindex/vacuum_while_reindex_ao_bitmap reindex/vacuum_while_reindex_heap_b test: cancel_plpython # Test concurrent UPDATE +test: concurrent_update test: concurrent_update_distkeys test: concurrent_update_epq diff --git a/src/test/regress/GNUmakefile b/src/test/regress/GNUmakefile index e19a468688..1677c5ee9e 100644 --- a/src/test/regress/GNUmakefile +++ b/src/test/regress/GNUmakefile @@ -176,10 +176,7 @@ query_info_hook_test: ## Run tests ## -REGRESS_OPTS = --dlpath=. --init-file=$(srcdir)/init_file $(EXTRA_REGRESS_OPTS) -ifeq ($(enable_faultinjector),yes) -REGRESS_OPTS += --load-extension=gp_inject_fault -endif +REGRESS_OPTS = --dlpath=. --init-file=$(srcdir)/init_file $(EXTRA_REGRESS_OPTS) --load-extension=gp_inject_fault check: all tablespace-setup $(pg_regress_check) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS) @@ -196,9 +193,6 @@ installcheck-small: all installcheck-good: all twophase_pqexecparams hooktest query_info_hook_test $(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule --schedule=$(srcdir)/greenplum_schedule $(EXTRA_TESTS) -ifeq ($(enable_faultinjector),yes) - $(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/greenplum_faultinjector_schedule $(EXTRA_TESTS) -endif installcheck-parallel: all tablespace-setup $(pg_regress_installcheck) $(REGRESS_OPTS) --schedule=$(srcdir)/parallel_schedule $(MAXCONNOPT) $(EXTRA_TESTS) diff --git a/src/test/regress/expected/alter_extension.out b/src/test/regress/expected/alter_extension.out index ea70ff2201..dd5338c69c 100644 --- a/src/test/regress/expected/alter_extension.out +++ b/src/test/regress/expected/alter_extension.out @@ -1,10 +1,11 @@ --- Creating extension to test alter extension --- Assume: pageinspect is shipped by default -CREATE EXTENSION pageinspect; +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; CREATE AGGREGATE example_agg(int4) ( SFUNC = int4larger, STYPE = int4 ); -ALTER EXTENSION pageinspect ADD AGGREGATE example_agg(int4); -ALTER EXTENSION pageinspect DROP AGGREGATE example_agg(int4); -DROP EXTENSION pageinspect; +ALTER EXTENSION gp_inject_fault ADD AGGREGATE example_agg(int4); +ALTER EXTENSION gp_inject_fault DROP AGGREGATE example_agg(int4); +-- Test creating an extension that already exists. Nothing too exciting about +-- it, but let's keep up the test coverage. +CREATE EXTENSION gp_inject_fault; +ERROR: extension "gp_inject_fault" already exists diff --git a/src/test/regress/expected/psql_gp_commands.out b/src/test/regress/expected/psql_gp_commands.out index 7e937b595b..2a64c18529 100644 --- a/src/test/regress/expected/psql_gp_commands.out +++ b/src/test/regress/expected/psql_gp_commands.out @@ -1,23 +1,26 @@ -- -- Test \dx and \dx+, to display extensions. -- --- We just use plpgsql as an example of an extension here. -\dx plpgsql - List of installed extensions - Name | Version | Schema | Description ----------+---------+------------+------------------------------ - plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language +-- We just use gp_inject_fault as an example of an extension here. We don't +-- inject any faults. +\dx gp_inject* + List of installed extensions + Name | Version | Schema | Description +-----------------+---------+--------+---------------------------------------------- + gp_inject_fault | 1.0 | public | simulate various faults for testing purposes (1 row) -\dx+ plpgsql - Objects in extension "plpgsql" - Object Description -------------------------------------------- - function plpgsql_call_handler() - function plpgsql_inline_handler(internal) - function plpgsql_validator(oid) - language plpgsql -(4 rows) +\dx+ gp_inject* + Objects in extension "gp_inject_fault" + Object Description +------------------------------------------------------------------------------------ + function force_mirrors_to_catch_up() + function gp_inject_fault(text,text,integer) + function gp_inject_fault(text,text,text,text,text,integer,integer,integer,integer) + function gp_inject_fault_infinite(text,text,integer) + function gp_wait_until_triggered_fault(text,integer,integer) + function insert_noop_xlog_record() +(6 rows) -- -- Test extended \du flags diff --git a/src/test/regress/greenplum_faultinjector_schedule b/src/test/regress/greenplum_faultinjector_schedule deleted file mode 100644 index 8e2ef0a708..0000000000 --- a/src/test/regress/greenplum_faultinjector_schedule +++ /dev/null @@ -1,26 +0,0 @@ -# This schedule file only contains tests that depend on fault injector - -test: createdb -test: spi_processed64bit -test: python_processed64bit -test: gp_tablespace_with_faults -test: gp_tablespace -test: gpcopy -test: toast -test: guc_gp -test: bitmap_index -test: dispatch -# test workfiles compressed using zlib -test: zlib -test: segspace -test: cursor -test: query_finish_pending -test: dtm_retry -test: aocs -test: ic -test: gporca_faults -test: fts_error -test: alter_db_set_tablespace -test: fts_recovery_in_progress -test: autovacuum-template0 - diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index e9051cefcf..65849858f3 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -26,14 +26,24 @@ test: instr_in_shmem_setup # run separately - because slot counter may influenced by other parallel queries test: instr_in_shmem +test: createdb test: gp_aggregates gp_aggregates_costs gp_metadata variadic_parameters default_parameters function_extensions spi gp_xml shared_scan update_gp returning_gp resource_queue_with_rule gp_types gp_index +test: spi_processed64bit +test: python_processed64bit +test: gp_tablespace_with_faults +# below test(s) inject faults so each of them need to be in a separate group +test: gp_tablespace test: temp_tablespaces test: default_tablespace test: leastsquares opr_sanity_gp decode_expr bitmapscan bitmapscan_ao case_gp limit_gp notin percentile join_gp union_gp gpcopy_encoding gp_create_table gp_create_view window_views replication_slots create_table_like_gp gp_constraints matview_ao gpcopy_dispatch +# below test(s) inject faults so each of them need to be in a separate group +test: gpcopy -test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans misc_jiras +test: filter gpctas gpdist gpdist_opclasses gpdist_legacy_opclasses matrix toast sublink table_functions olap_setup complex opclass_ddl information_schema guc_env_var gp_explain distributed_transactions explain_format olap_plans misc_jiras +# below test(s) inject faults so each of them need to be in a separate group +test: guc_gp # namespace_gp test will show diff if concurrent tests use temporary tables. # So run it separately. @@ -42,8 +52,12 @@ test: namespace_gp # test gpdb internal and segment connections test: gp_connections +# bitmap_index triggers recovery, run it seperately +test: bitmap_index test: gp_dump_query_oids analyze gp_owner_permission incremental_analyze test: indexjoin as_alias regex_gp gpparams with_clause transient_types gp_rules dispatch_encoding motion_gp +# dispatch should always run seperately from other cases. +test: dispatch # interconnect tests test: icudp/gp_interconnect_queue_depth icudp/gp_interconnect_queue_depth_longtime icudp/gp_interconnect_snd_queue_depth icudp/gp_interconnect_snd_queue_depth_longtime icudp/gp_interconnect_min_retries_before_timeout icudp/gp_interconnect_transmit_timeout icudp/gp_interconnect_cache_future_packets icudp/gp_interconnect_default_rtt icudp/gp_interconnect_fc_method icudp/gp_interconnect_min_rto icudp/gp_interconnect_timer_checking_period icudp/gp_interconnect_timer_period icudp/queue_depth_combination_loss icudp/queue_depth_combination_capacity @@ -57,14 +71,29 @@ test: deadlock2 # test workfiles test: workfile/hashagg_spill workfile/hashjoin_spill workfile/materialize_spill workfile/sisc_mat_sort workfile/sisc_sort_spill workfile/sort_spill workfile/spilltodisk +# test workfiles compressed using zlib +# 'zlib' utilizes fault injectors so it needs to be in a group by itself +test: zlib # Check for shmem leak for instrumentation slots before gpdb restart test: instr_in_shmem_verify test: workfile_limits +# It will also use faultinjector - so it needs to be in a group by itself. +test: segspace + +test: cursor + +# 'query_finish_pending' sets QueryFinishPending flag to true during query execution using fault injectors +# so it needs to be in a group by itself +test: query_finish_pending test: gpdiffcheck gptokencheck gp_hashagg sequence_gp tidscan_gp co_nestloop_idxscan dml_in_udf gpdtm_plpgsql +# The test must be run by itself as it injects a fault on QE to fail +# at the 2nd phase of 2PC. +test: dtm_retry + # The appendonly test cannot be run concurrently with tests that have # serializable transactions (may conflict with AO vacuum operations). test: rangefuncs_cdb gp_dqa subselect_gp subselect_gp2 gp_transactions olap_group olap_window_seq sirv_functions appendonly create_table_distpol alter_distpol_dropped query_finish partial_table subselect_gp_indexes @@ -94,6 +123,9 @@ test: db_size_functions ignore: gp_portal_error test: external_table external_table_create_privs external_table_persistent_error_log column_compression eagerfree alter_table_aocs alter_table_aocs2 alter_distribution_policy aoco_privileges test: alter_table_set alter_table_gp alter_table_ao subtransaction_visibility oid_consistency udf_exception_blocks +# below test(s) inject faults so each of them need to be in a separate group +test: aocs +test: ic test: resource_queue test: resource_queue_function @@ -112,6 +144,9 @@ test: direct_dispatch bfv_dd bfv_dd_multicolumn bfv_dd_types test: bfv_catalog bfv_index bfv_olap bfv_aggregate bfv_partition bfv_partition_plans DML_over_joins bfv_statistic nested_case_null sort bb_mpph aggregate_with_groupingsets gporca +# NOTE: gporca_faults uses gp_fault_injector - so do not add to a parallel group +test: gporca_faults + test: bb_memory_quota # Tests for replicated table @@ -195,6 +230,8 @@ test: vacuum_full_heap_bitmapindex test: ao_checksum_corruption AOCO_Compression AORO_Compression table_statistics test: workfile_mgr_test test: session_reset +# below test(s) inject faults so each of them need to be in a separate group +test: fts_error test: psql_gp_commands pg_resetxlog dropdb_check_shared_buffer_cache gp_upgrade_cornercases @@ -202,12 +239,18 @@ test: psql_gp_commands pg_resetxlog dropdb_check_shared_buffer_cache gp_upgrade_ test: instr_in_shmem_verify test: temp_relation +test: alter_db_set_tablespace # This cannot run in parallel because other tests could increment the Oid # counters and make the Oid counter observations hardcoded in the answer file # incorrect. test: oid_wraparound +# fts_recovery_in_progresss uses fault injectors to simulate FTS fault states, +# hence it should be run in isolation. +test: fts_recovery_in_progress +test: autovacuum-template0 + # gpexpand introduce the partial tables, check them if they can run correctly test: gangsize gang_reuse diff --git a/src/test/regress/scan_flaky_fault_injectors.sh b/src/test/regress/scan_flaky_fault_injectors.sh index 2d42d706a2..94541270be 100755 --- a/src/test/regress/scan_flaky_fault_injectors.sh +++ b/src/test/regress/scan_flaky_fault_injectors.sh @@ -9,26 +9,13 @@ set -e fault_injection_tests=$(mktemp fault_injection_tests.XXX) parallel_tests=$(mktemp parallel_tests.XXX) -row_col_tests=$(mktemp row_col_tests.XXXX) retcode=0 -grep -ERIl '@orientation@' input \ -| sed 's,^[^/]*/\(.*\)\.[^.]*$,\1,' \ -| sort -u \ -> $row_col_tests - # list the tests that inject faults -grep -ERIli '(\s+gp_inject_fault)|force_mirrors_to_catch_up' sql input \ +grep -ERIli '(select|perform)\s+gp_inject_fault' sql input \ | sed 's,^[^/]*/\(.*\)\.[^.]*$,\1,' \ | sort -u \ > $fault_injection_tests -# source files under input may contain two types of tests: _row and _column -comm -23 $fault_injection_tests $row_col_tests > $parallel_tests -comm -12 $fault_injection_tests $row_col_tests \ -| sed 's,^\(.*\)$,\1_row \1_column,' \ -| tr ' ' '\n' \ ->> $parallel_tests -sort -u $parallel_tests > $fault_injection_tests echo "scanning for flaky fault-injection tests..." @@ -48,7 +35,7 @@ for schedule in *_schedule; do fi done -rm -f $fault_injection_tests $parallel_tests $row_col_tests +rm -f $fault_injection_tests $parallel_tests if [ $retcode = 0 ]; then echo "done" diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index 690130dc59..cdcdee5c8f 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -171,3 +171,4 @@ test: with test: xml test: event_trigger test: stats +test: createdb diff --git a/src/test/regress/sql/alter_extension.sql b/src/test/regress/sql/alter_extension.sql index 8475f7922d..5e28c0194c 100644 --- a/src/test/regress/sql/alter_extension.sql +++ b/src/test/regress/sql/alter_extension.sql @@ -1,13 +1,13 @@ --- Creating extension to test alter extension --- Assume: pageinspect is shipped by default -CREATE EXTENSION pageinspect; +CREATE EXTENSION IF NOT EXISTS gp_inject_fault; CREATE AGGREGATE example_agg(int4) ( SFUNC = int4larger, STYPE = int4 ); -ALTER EXTENSION pageinspect ADD AGGREGATE example_agg(int4); -ALTER EXTENSION pageinspect DROP AGGREGATE example_agg(int4); +ALTER EXTENSION gp_inject_fault ADD AGGREGATE example_agg(int4); +ALTER EXTENSION gp_inject_fault DROP AGGREGATE example_agg(int4); -DROP EXTENSION pageinspect; +-- Test creating an extension that already exists. Nothing too exciting about +-- it, but let's keep up the test coverage. +CREATE EXTENSION gp_inject_fault; diff --git a/src/test/regress/sql/psql_gp_commands.sql b/src/test/regress/sql/psql_gp_commands.sql index 29083e4da7..cbd130ece7 100644 --- a/src/test/regress/sql/psql_gp_commands.sql +++ b/src/test/regress/sql/psql_gp_commands.sql @@ -1,9 +1,11 @@ -- -- Test \dx and \dx+, to display extensions. -- --- We just use plpgsql as an example of an extension here. -\dx plpgsql -\dx+ plpgsql +-- We just use gp_inject_fault as an example of an extension here. We don't +-- inject any faults. + +\dx gp_inject* +\dx+ gp_inject* -- diff --git a/src/test/walrep/Makefile b/src/test/walrep/Makefile index 01c8739361..21dd709ac0 100644 --- a/src/test/walrep/Makefile +++ b/src/test/walrep/Makefile @@ -7,12 +7,8 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global REGRESS = setup -REGRESS += replication_views_mirrored walreceiver generate_ao_xlog generate_aoco_xlog - -ifeq ($(enable_faultinjector),yes) -REGRESS += missing_xlog +REGRESS += replication_views_mirrored missing_xlog walreceiver generate_ao_xlog generate_aoco_xlog REGRESS_OPTS = --load-extension=gp_inject_fault -endif NO_PGXS = 1 include $(top_srcdir)/src/makefiles/pgxs.mk -- GitLab