From 83c336ad91a2aec265d15e948863a19988385992 Mon Sep 17 00:00:00 2001 From: Asim R P Date: Mon, 12 Aug 2019 16:51:29 +0800 Subject: [PATCH] Replace old fault injector interface with the new one. The new interface employs a special libpq connection parameter and a libpq message to convey fault details to destination postmaster. This interface has been in use for some time now. It was getting cumbersome to maintain both the old as well as the new interface. To be clear, we now have only one interface to inject fault and that is "gp_inject_fault()". Existing tests that were using gp_inject_fault2() will be updated in a follow up commit. --- .../expected/inject_fault_test.out | 58 +++++++-------- .../gp_inject_fault/gp_inject_fault--1.0.sql | 52 +------------ gpcontrib/gp_inject_fault/gp_inject_fault.c | 74 +------------------ .../gp_inject_fault/sql/inject_fault_test.sql | 8 +- 4 files changed, 36 insertions(+), 156 deletions(-) diff --git a/gpcontrib/gp_inject_fault/expected/inject_fault_test.out b/gpcontrib/gp_inject_fault/expected/inject_fault_test.out index baa04e9682..cc76128b41 100644 --- a/gpcontrib/gp_inject_fault/expected/inject_fault_test.out +++ b/gpcontrib/gp_inject_fault/expected/inject_fault_test.out @@ -4,59 +4,53 @@ CREATE EXTENSION gp_inject_fault; begin; -- inject fault of type sleep on all primaries select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'sleep', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'sleep', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -NOTICE: Success: -NOTICE: Success: -NOTICE: Success: gp_inject_fault ----------------- - t - t - t + Success: + Success: + Success: (3 rows) -- check fault status select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'status', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'status', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'set' num times hit:'0' -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'set' num times hit:'0' -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'set' num times hit:'0' - gp_inject_fault ------------------ - t - t - t + gp_inject_fault +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + (3 rows) -- commit transaction should trigger the fault end; -- fault status should indicate it's triggered select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'status', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'status', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'completed' num times hit:'1' -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'completed' num times hit:'1' -NOTICE: Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' occurrence:'1' sleep time:'2' fault injection state:'completed' num times hit:'1' - gp_inject_fault ------------------ - t - t - t + gp_inject_fault +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + + Success: fault name:'finish_prepared_after_record_commit_prepared' fault type:'sleep' ddl statement:'' database name:'' table name:'' start occurrence:'1' end occurrence:'2' extra arg:'0' fault injection state:'set' num times hit:'0' + + (3 rows) -- reset the fault on all primaries select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'reset', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'reset', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -NOTICE: Success: -NOTICE: Success: -NOTICE: Success: gp_inject_fault ----------------- - t - t - t + Success: + Success: + Success: (3 rows) diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault--1.0.sql b/gpcontrib/gp_inject_fault/gp_inject_fault--1.0.sql index b5dc9e1e49..2ba7165309 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault--1.0.sql +++ b/gpcontrib/gp_inject_fault/gp_inject_fault--1.0.sql @@ -13,7 +13,7 @@ CREATE FUNCTION gp_inject_fault( end_occurrence int4, extra_arg int4, db_id int4) -RETURNS boolean +RETURNS text AS 'MODULE_PATHNAME' LANGUAGE C VOLATILE STRICT NO SQL; @@ -23,7 +23,7 @@ CREATE FUNCTION gp_inject_fault( faultname text, type text, db_id int4) -RETURNS boolean +RETURNS text AS $$ select gp_inject_fault($1, $2, '', '', '', 1, 1, 0, $3) $$ LANGUAGE SQL; @@ -32,7 +32,7 @@ CREATE FUNCTION gp_inject_fault_infinite( faultname text, type text, db_id int4) -RETURNS boolean +RETURNS text AS $$ select gp_inject_fault($1, $2, '', '', '', 1, -1, 0, $3) $$ LANGUAGE SQL; @@ -43,50 +43,6 @@ CREATE FUNCTION gp_wait_until_triggered_fault( faultname text, numtimestriggered int4, db_id int4) -RETURNS boolean -AS $$ select gp_inject_fault($1, 'wait_until_triggered', '', '', '', 1, 1, $2, $3) $$ -LANGUAGE SQL; - -CREATE FUNCTION gp_inject_fault2( - faultname text, - type text, - ddl text, - database text, - tablename text, - start_occurrence int4, - end_occurrence int4, - extra_arg int4, - db_id int4) RETURNS text -AS 'MODULE_PATHNAME' -LANGUAGE C VOLATILE STRICT NO SQL; - --- Simpler version, trigger only one time, occurrence start at 1 and --- end at 1, no sleep and no ddl/database/tablename. -CREATE FUNCTION gp_inject_fault2( - faultname text, - type text, - db_id int4) -RETURNS text -AS $$ select gp_inject_fault2($1, $2, '', '', '', 1, 1, 0, $3) $$ -LANGUAGE SQL; - --- Simpler version, always trigger until fault is reset. -CREATE FUNCTION gp_inject_fault_infinite2( - faultname text, - type text, - db_id int4) -RETURNS text -AS $$ select gp_inject_fault2($1, $2, '', '', '', 1, -1, 0, $3) $$ -LANGUAGE SQL; - --- Simpler version to avoid confusion for wait_until_triggered fault. --- occurrence in call below defines wait until number of times the --- fault hits. -CREATE FUNCTION gp_wait_until_triggered_fault2( - faultname text, - numtimestriggered int4, - db_id int4) -RETURNS text -AS $$ select gp_inject_fault2($1, 'wait_until_triggered', '', '', '', 1, 1, $2, $3) $$ +AS $$ select gp_inject_fault($1, 'wait_until_triggered', '', '', '', 1, 1, $2, $3) $$ LANGUAGE SQL; diff --git a/gpcontrib/gp_inject_fault/gp_inject_fault.c b/gpcontrib/gp_inject_fault/gp_inject_fault.c index cd3429e58f..342b7a5d1e 100644 --- a/gpcontrib/gp_inject_fault/gp_inject_fault.c +++ b/gpcontrib/gp_inject_fault/gp_inject_fault.c @@ -20,7 +20,6 @@ PG_MODULE_MAGIC; extern Datum gp_inject_fault(PG_FUNCTION_ARGS); -extern Datum gp_inject_fault2(PG_FUNCTION_ARGS); void _PG_init(void); static void @@ -48,74 +47,6 @@ _PG_init(void) MemoryContextSwitchTo(oldContext); } -PG_FUNCTION_INFO_V1(gp_inject_fault); -Datum -gp_inject_fault(PG_FUNCTION_ARGS) -{ - char *faultName = TextDatumGetCString(PG_GETARG_DATUM(0)); - char *type = TextDatumGetCString(PG_GETARG_DATUM(1)); - char *ddlStatement = TextDatumGetCString(PG_GETARG_DATUM(2)); - char *databaseName = TextDatumGetCString(PG_GETARG_DATUM(3)); - char *tableName = TextDatumGetCString(PG_GETARG_DATUM(4)); - int startOccurrence = PG_GETARG_INT32(5); - int endOccurrence = PG_GETARG_INT32(6); - int extraArg = PG_GETARG_INT32(7); - int dbid = PG_GETARG_INT32(8); - - /* Fast path if injecting fault in our postmaster. */ - if (GpIdentity.dbid == dbid) - { - char *response; - - response = InjectFault( - faultName, type, ddlStatement, databaseName, - tableName, startOccurrence, endOccurrence, extraArg); - if (!response) - elog(ERROR, "failed to inject fault locally (dbid %d)", dbid); - if (strncmp(response, "Success:", strlen("Success:")) != 0) - elog(ERROR, "%s", response); - - elog(NOTICE, "%s", response); - } - else if (Gp_role == GP_ROLE_DISPATCH) - { - /* - * Otherwise, relay the command to executor nodes. - * - * We'd only really need to dispatch it to the one that it's meant for, - * but for now, just send it everywhere. The other nodes will just - * ignore it. - * - * (Perhaps this function should be defined as EXECUTE ON SEGMENTS, - * instead of dispatching manually here? But then it wouldn't run on - * QD. There is no EXECUTE ON SEGMENTS AND MASTER options, at the - * moment...) - * - * NOTE: Because we use the normal dispatcher to send this query, - * if a fault has already been injected to the dispatcher code, - * this will trigger it. That means that if you wish to inject - * faults on both the dispatcher and an executor in the same test, - * you need to be careful with the order you inject the faults! - */ - char *sql; - - sql = psprintf("select gp_inject_fault(%s, %s, %s, %s, %s, %d, %d, %d, %d)", - quote_literal_cstr(faultName), - quote_literal_cstr(type), - quote_literal_cstr(ddlStatement), - quote_literal_cstr(databaseName), - quote_literal_cstr(tableName), - startOccurrence, - endOccurrence, - extraArg, - dbid); - - CdbDispatchCommand(sql, DF_CANCEL_ON_ERROR, NULL); - } - PG_RETURN_DATUM(BoolGetDatum(true)); -} - - static void getHostnameAndPort(int dbid, char **hostname, int *port) { @@ -155,10 +86,9 @@ getHostnameAndPort(int dbid, char **hostname, int *port) heap_close(configrel, NoLock); } - -PG_FUNCTION_INFO_V1(gp_inject_fault2); +PG_FUNCTION_INFO_V1(gp_inject_fault); Datum -gp_inject_fault2(PG_FUNCTION_ARGS) +gp_inject_fault(PG_FUNCTION_ARGS) { char *faultName = TextDatumGetCString(PG_GETARG_DATUM(0)); char *type = TextDatumGetCString(PG_GETARG_DATUM(1)); diff --git a/gpcontrib/gp_inject_fault/sql/inject_fault_test.sql b/gpcontrib/gp_inject_fault/sql/inject_fault_test.sql index ea82040360..5159478b39 100644 --- a/gpcontrib/gp_inject_fault/sql/inject_fault_test.sql +++ b/gpcontrib/gp_inject_fault/sql/inject_fault_test.sql @@ -5,19 +5,19 @@ CREATE EXTENSION gp_inject_fault; begin; -- inject fault of type sleep on all primaries select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'sleep', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'sleep', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -- check fault status select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'status', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'status', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -- commit transaction should trigger the fault end; -- fault status should indicate it's triggered select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'status', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'status', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -- reset the fault on all primaries select gp_inject_fault('finish_prepared_after_record_commit_prepared', - 'reset', '', '', '', 1, 2, dbid) from gp_segment_configuration + 'reset', '', '', '', 1, 2, 0, dbid) from gp_segment_configuration where role = 'p' and content > -1; -- GitLab