From 825ca1e3e6cc62e20a42fedc3718138e0f53b8b4 Mon Sep 17 00:00:00 2001 From: Taylor Vesely Date: Fri, 24 Aug 2018 14:00:55 -0700 Subject: [PATCH] Add DEBUG mode to the explain_memory_verbosity GUC The memory accounting system generates a new memory account for every execution node initialized in ExecInitNode. The address to these memory accounts is stored in the shortLivingMemoryAccountArray. If the memory allocated for shortLivingMemoryAccountArray is full, we will repalloc the array with double the number of available entries. After creating approximately 67000000 memory accounts, it will need to allocate more than 1GB of memory to increase the array size, and throw an ERROR, canceling the running query. PL/pgSQL and SQL functions will create new executors/plan nodes that must be tracked my the memory accounting system. This level of detail is not necessary for tracking memory leaks, and creating a separate memory account for every executor will use large amount of memory just to track these memory accounts. Instead of tracking millions of individual memory accounts, we consolidate any child executor account into a special 'X_NestedExecutor' account. If explain_memory_verbosity is set to 'detailed' and below, consolidate all child executors into this account. If more detail is needed for debugging, set explain_memory_verbosity to 'debug', where, as was the previous behavior, every executor will be assigned its own MemoryAccountId. Originally we tried to remove nested execution accounts after they finish executing, but rolling over those accounts into a 'X_NestedExecutor' account was impracticable to accomplish without the possibility of a future regression. If any accounts are created between nested executors that are not rolled over to an 'X_NestedExecutor' account, recording which accounts are rolled over can grow in the same way that the shortLivingMemoryAccountArray is growing today, and would also grow too large to reasonably fit in memory. If we were to iterate through the SharedHeaders every time that we finish a nested executor, it is not likely to be very performant. While we were at it, convert some of the convenience macros dealing with memory accounting for executor / planner node into functions, and move them out of memory accounting header files into the sole callers' compilation units. Co-authored-by: Ashwin Agrawal Co-authored-by: Ekta Khanna Co-authored-by: Adam Berlin Co-authored-by: Joao Pereira Co-authored-by: Melanie Plageman --- src/backend/executor/execMain.c | 2 +- src/backend/executor/execProcnode.c | 52 ++++++- src/backend/optimizer/plan/planner.c | 10 +- src/backend/utils/misc/guc_gp.c | 3 +- src/backend/utils/mmgr/memaccounting.c | 104 +++++++++++++ .../utils/mmgr/test/memaccounting_test.c | 2 +- src/include/utils/memaccounting.h | 67 +++++--- src/test/regress/expected/memconsumption.out | 143 ++++++++++++++++++ src/test/regress/sql/memconsumption.sql | 86 +++++++++++ 9 files changed, 443 insertions(+), 26 deletions(-) diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index aeb2c70e39..879df839d4 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -267,7 +267,7 @@ standard_ExecutorStart(QueryDesc *queryDesc, int eflags) PlannedStmt *plannedStmt = queryDesc->plannedstmt; - queryDesc->memoryAccountId = MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_EXECUTOR); + queryDesc->memoryAccountId = MemoryAccounting_CreateExecutorMemoryAccount(); START_MEMORY_ACCOUNT(queryDesc->memoryAccountId); diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c index 2c2220263c..8f7981f0ab 100644 --- a/src/backend/executor/execProcnode.c +++ b/src/backend/executor/execProcnode.c @@ -159,6 +159,54 @@ static CdbVisitOpt planstate_walk_kids(PlanState *planstate, void *context, int flags); +/* + * saveExecutorMemoryAccount saves an operator specific memory account + * into the PlanState of that operator + */ +static inline void +saveExecutorMemoryAccount(PlanState *execState, + MemoryAccountIdType curMemoryAccountId) +{ + Assert(curMemoryAccountId != MEMORY_OWNER_TYPE_Undefined); + Assert(MEMORY_OWNER_TYPE_Undefined == execState->memoryAccountId); + execState->memoryAccountId = curMemoryAccountId; +} + + +/* + * CREATE_EXECUTOR_MEMORY_ACCOUNT is a convenience macro to create a new + * operator specific memory account *if* the operator will be executed in + * the current slice, i.e., it is not part of some other slice (alien + * plan node). We assign a shared AlienExecutorMemoryAccount for plan nodes + * that will not be executed in current slice + * + * If the operator is a child of an 'X_NestedExecutor' account, the operator + * is also assigned to the 'X_NestedExecutor' account, unless the + * explain_memory_verbosity guc is set to 'debug' or above. + */ +#define CREATE_EXECUTOR_MEMORY_ACCOUNT(isAlienPlanNode, planNode, NodeType) \ + MemoryAccounting_CreateExecutorAccountWithType( \ + (isAlienPlanNode), (planNode), MEMORY_OWNER_TYPE_Exec_##NodeType) + +static inline MemoryAccountIdType +MemoryAccounting_CreateExecutorAccountWithType(bool isAlienPlanNode, + Plan *node, + MemoryOwnerType ownerType) +{ + if (isAlienPlanNode) + return MEMORY_OWNER_TYPE_Exec_AlienShared; + else + if (MemoryAccounting_IsUnderNestedExecutor() && + explain_memory_verbosity < EXPLAIN_MEMORY_VERBOSITY_DEBUG) + return MemoryAccounting_GetOrCreateNestedExecutorAccount(); + else + { + long maxLimit = + node->operatorMemKB == 0 ? work_mem : node->operatorMemKB; + return MemoryAccounting_CreateAccount(maxLimit, ownerType); + } +} + /* * setSubplanSliceId * Set the slice id info for the given subplan. @@ -219,7 +267,7 @@ ExecInitNode(Plan *node, EState *estate, int eflags) int origSliceIdInPlan = estate->currentSliceIdInPlan; int origExecutingSliceId = estate->currentExecutingSliceId; - MemoryAccountIdType curMemoryAccountId = MEMORY_OWNER_TYPE_Undefined; + MemoryAccountIdType curMemoryAccountId; int localMotionId = LocallyExecutingSliceIndex(estate); @@ -835,7 +883,7 @@ ExecInitNode(Plan *node, EState *estate, int eflags) if (result != NULL) { - SAVE_EXECUTOR_MEMORY_ACCOUNT(result, curMemoryAccountId); + saveExecutorMemoryAccount(result, curMemoryAccountId); } return result; } diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index c8ce5f6d3f..c4169badce 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -200,6 +200,7 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) PlannerConfig *config; instr_time starttime; instr_time endtime; + MemoryAccountIdType curMemoryAccountId; /* * Use ORCA only if it is enabled and we are in a master QD process. @@ -219,7 +220,10 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) if (gp_log_optimization_time) INSTR_TIME_SET_CURRENT(starttime); - START_MEMORY_ACCOUNT(MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_Optimizer)); + curMemoryAccountId = MemoryAccounting_CreatePlanningMemoryAccount( + MEMORY_OWNER_TYPE_Optimizer); + + START_MEMORY_ACCOUNT(curMemoryAccountId); { result = optimize_query(parse, boundParams); } @@ -243,11 +247,13 @@ standard_planner(Query *parse, int cursorOptions, ParamListInfo boundParams) if (gp_log_optimization_time) INSTR_TIME_SET_CURRENT(starttime); + curMemoryAccountId = + MemoryAccounting_CreatePlanningMemoryAccount(MEMORY_OWNER_TYPE_Planner); /* * Incorrectly indented on purpose to avoid re-indenting an entire upstream * function */ - START_MEMORY_ACCOUNT(MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_Planner)); + START_MEMORY_ACCOUNT(curMemoryAccountId); { /* Cursor options may come from caller or from DECLARE CURSOR stmt */ diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index 894f54cb20..a041cd6261 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -530,6 +530,7 @@ static const struct config_enum_entry explain_memory_verbosity_options[] = { {"suppress", EXPLAIN_MEMORY_VERBOSITY_SUPPRESS}, {"summary", EXPLAIN_MEMORY_VERBOSITY_SUMMARY}, {"detail", EXPLAIN_MEMORY_VERBOSITY_DETAIL}, + {"debug", EXPLAIN_MEMORY_VERBOSITY_DEBUG}, {NULL, 0} }; @@ -4962,7 +4963,7 @@ struct config_enum ConfigureNamesEnum_gp[] = { {"explain_memory_verbosity", PGC_USERSET, RESOURCES_MEM, gettext_noop("Experimental feature: show memory account usage in EXPLAIN ANALYZE."), - gettext_noop("Valid values are SUPPRESS, SUMMARY, and DETAIL."), + gettext_noop("Valid values are SUPPRESS, SUMMARY, DETAIL, and DEBUG."), GUC_GPDB_ADDOPT }, &explain_memory_verbosity, diff --git a/src/backend/utils/mmgr/memaccounting.c b/src/backend/utils/mmgr/memaccounting.c index 1ba64cfda9..09565a9025 100644 --- a/src/backend/utils/mmgr/memaccounting.c +++ b/src/backend/utils/mmgr/memaccounting.c @@ -73,6 +73,35 @@ MemoryAccountIdType liveAccountStartId = MEMORY_OWNER_TYPE_START_SHORT_LIVING; */ MemoryAccountIdType nextAccountId = MEMORY_OWNER_TYPE_START_SHORT_LIVING; +/* + * The memory account for the primary executor account. This will be assigned to the + * first executor created during query execution. + */ +MemoryAccountIdType mainExecutorAccount = MEMORY_OWNER_TYPE_Undefined; + +/* + * MemoryAccountId for the 'X_NestedExecutor' account. This account is a direct + * child of the mainExecutorAccount. This is intended for accounts created + * during a "nested" ExecutorStart (indirectly called by ExecutorRun), anything + * that would normally create another account should instead collapse into the + * mainNestedExecutorAccount. + * + * You would think that it would be okay to create short living memory accounts + * then roll them over to the X_NestedExecutor account in the same way that we + * use the Rollover account. Unfortunately, this does not work because we need + * a way to to keep track of which account IDs have been rolled over. In the + * case of the Rollover account, we 'retire' any old memory account by calling + * AdvanceMemoryAccountingGeneration() to retire all accounts older than the + * liveAccountStartId, but when using live accounts the problem is a lot more + * complicated. Keeping a list of 'retired' shortLivingMemoryAccounts does not + * solve the problem, because the list of retired accounts is unbounded. Even + * if we store the list of accounts as a vector, it does not necessarily + * resolve the issue, because it is possible to have a 'hole' in the list of + * accounts, and we shouldn't assume that all of the retired accounts are + * contiguous. + */ +MemoryAccountIdType mainNestedExecutorAccount= MEMORY_OWNER_TYPE_Undefined; + /* ****************************************************** * Internal methods declarations @@ -1188,6 +1217,8 @@ MemoryAccounting_GetOwnerName(MemoryOwnerType ownerType) return "X_WorkTableScan"; case MEMORY_OWNER_TYPE_Exec_ForeignScan: return "X_ForeignScan"; + case MEMORY_OWNER_TYPE_Exec_NestedExecutor: + return "X_NestedExecutor"; default: Assert(false); break; @@ -1431,6 +1462,16 @@ AdvanceMemoryAccountingGeneration() liveAccountStartId = nextAccountId; + /* + * Currently this is the only place where we reset the mainExecutorAccount. + * XXX: We have a flawed assumption that the first "executor" account is + * the main one and that all subsequently created "executor" accounts are + * nested. Rewrite rules and constant folding during planning are examples + * of false detection of nested executor. + */ + mainExecutorAccount = MEMORY_OWNER_TYPE_Undefined; + mainNestedExecutorAccount = MEMORY_OWNER_TYPE_Undefined; + Assert(RolloverMemoryAccount->peak >= MemoryAccountingPeakBalance); } @@ -1472,3 +1513,66 @@ SaveMemoryBufToDisk(struct StringInfoData *memoryBuf, char *prefix) fclose(file); } + +/* + * CreateMainExecutor + * Creates the mainExecutorAccount + */ +MemoryAccountIdType +MemoryAccounting_CreateMainExecutor() +{ + Assert(mainExecutorAccount == MEMORY_OWNER_TYPE_Undefined); + mainExecutorAccount = MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_EXECUTOR); + + return mainExecutorAccount; +} + + +/* + * GetOrCreateNestedExecutorAccount + * Returns the MemoryAccountId for the 'X_NestedExecutor' account. Creates the + * mainNestedExecutorAccount if it does not already exist. + */ +MemoryAccountIdType +MemoryAccounting_GetOrCreateNestedExecutorAccount() +{ + if (mainNestedExecutorAccount == MEMORY_OWNER_TYPE_Undefined) + { + Assert(mainExecutorAccount != MEMORY_OWNER_TYPE_Undefined); + START_MEMORY_ACCOUNT(mainExecutorAccount); + mainNestedExecutorAccount = MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_Exec_NestedExecutor); + END_MEMORY_ACCOUNT(); + } + return mainNestedExecutorAccount; +} + +/* + * IsUnderNestedExecutor + * Return weather the currently active memory account is 'X_NestedExecutor'. + */ +bool +MemoryAccounting_IsUnderNestedExecutor() +{ + return ActiveMemoryAccountId == mainNestedExecutorAccount; +} + +/* + * IsMainExecutorCreated + * Returns whether the top level executor account has been created. + */ +bool +MemoryAccounting_IsMainExecutorCreated() +{ + return mainExecutorAccount != MEMORY_OWNER_TYPE_Undefined; +} + +MemoryAccountIdType +MemoryAccounting_CreatePlanningMemoryAccount(MemoryOwnerType type) +{ + if (explain_memory_verbosity >= EXPLAIN_MEMORY_VERBOSITY_DEBUG) + return MemoryAccounting_CreateAccount(0, type); + else if (MemoryAccounting_IsMainExecutorCreated()) + return MemoryAccounting_GetOrCreateNestedExecutorAccount(); + else + return MemoryAccounting_CreateAccount(0, type); +} diff --git a/src/backend/utils/mmgr/test/memaccounting_test.c b/src/backend/utils/mmgr/test/memaccounting_test.c index 0560531fa1..9d864fef59 100755 --- a/src/backend/utils/mmgr/test/memaccounting_test.c +++ b/src/backend/utils/mmgr/test/memaccounting_test.c @@ -961,7 +961,7 @@ test__MemoryAccounting_GetAccountName__Validate(void **state) "X_BitmapHeapScan", "X_BitmapAppendOnlyScan", "X_TidScan", "X_SubqueryScan", "X_FunctionScan", "X_TableFunctionScan", "X_ValuesScan", "X_NestLoop", "X_MergeJoin", "X_HashJoin", "X_Material", "X_Sort", "X_Agg", "X_Unique", "X_Hash", "X_SetOp", "X_Limit", "X_Motion", "X_ShareInputScan", "X_WindowAgg", "X_Repeat", "X_ModifyTable", "X_LockRows", "X_DML", "X_SplitUpdate", "X_RowTrigger", - "X_AssertOp", "X_BitmapTableScan", "X_PartitionSelector", "X_RecursiveUnion", "X_CteScan", "X_WorkTableScan", "X_ForeignScan"}; + "X_AssertOp", "X_BitmapTableScan", "X_PartitionSelector", "X_RecursiveUnion", "X_CteScan", "X_WorkTableScan", "X_ForeignScan", "X_NestedExecutor"}; /* Ensure we have all the long living accounts in the longLivingNames array */ assert_true(sizeof(longLivingNames) / sizeof(char*) == MEMORY_OWNER_TYPE_END_LONG_LIVING); diff --git a/src/include/utils/memaccounting.h b/src/include/utils/memaccounting.h index cd4b3a07bd..975f943c95 100644 --- a/src/include/utils/memaccounting.h +++ b/src/include/utils/memaccounting.h @@ -25,6 +25,8 @@ struct MemoryContextData; #define EXPLAIN_MEMORY_VERBOSITY_SUPPRESS 0 /* Suppress memory reporting in explain analyze */ #define EXPLAIN_MEMORY_VERBOSITY_SUMMARY 1 /* Summary of memory usage for each owner in explain analyze */ #define EXPLAIN_MEMORY_VERBOSITY_DETAIL 2 /* Detail memory accounting tree for each slice in explain analyze */ +#define EXPLAIN_MEMORY_VERBOSITY_DEBUG 3 /* Detail memory accounting tree with every executor having its own account + * for each slice in explain analyze */ /* * MemoryAccount is a private data structure for recording memory usage by @@ -151,7 +153,8 @@ typedef enum MemoryOwnerType MEMORY_OWNER_TYPE_Exec_CteScan, MEMORY_OWNER_TYPE_Exec_WorkTableScan, MEMORY_OWNER_TYPE_Exec_ForeignScan, - MEMORY_OWNER_TYPE_EXECUTOR_END = MEMORY_OWNER_TYPE_Exec_ForeignScan, + MEMORY_OWNER_TYPE_Exec_NestedExecutor, + MEMORY_OWNER_TYPE_EXECUTOR_END = MEMORY_OWNER_TYPE_Exec_NestedExecutor, MEMORY_OWNER_TYPE_END_SHORT_LIVING = MEMORY_OWNER_TYPE_EXECUTOR_END } MemoryOwnerType; @@ -186,25 +189,7 @@ extern MemoryAccountIdType ActiveMemoryAccountId; ActiveMemoryAccountId = oldActiveMemoryAccountId;\ } while (0); -/* - * CREATE_EXECUTOR_MEMORY_ACCOUNT is a convenience macro to create a new - * operator specific memory account *if* the operator will be executed in - * the current slice, i.e., it is not part of some other slice (alien - * plan node). We assign a shared AlienExecutorMemoryAccount for plan nodes - * that will not be executed in current slice - */ -#define CREATE_EXECUTOR_MEMORY_ACCOUNT(isAlienPlanNode, planNode, NodeType) \ - isAlienPlanNode ? MEMORY_OWNER_TYPE_Exec_AlienShared : \ - MemoryAccounting_CreateAccount(((Plan*)node)->operatorMemKB == 0 ? \ - work_mem : ((Plan*)node)->operatorMemKB, MEMORY_OWNER_TYPE_Exec_##NodeType); -/* - * SAVE_EXECUTOR_MEMORY_ACCOUNT saves an operator specific memory account - * into the PlanState of that operator - */ -#define SAVE_EXECUTOR_MEMORY_ACCOUNT(execState, curMemoryAccountId)\ - Assert(MEMORY_OWNER_TYPE_Undefined == ((PlanState *)execState)->memoryAccountId); \ - ((PlanState *)execState)->memoryAccountId = curMemoryAccountId; extern MemoryAccountIdType MemoryAccounting_CreateAccount(long maxLimit, enum MemoryOwnerType ownerType); @@ -252,4 +237,48 @@ MemoryAccounting_RequestQuotaIncrease(void); extern MemoryAccountExplain * MemoryAccounting_ExplainCurrentOptimizerAccountInfo(void); +extern MemoryAccountIdType +MemoryAccounting_CreateMainExecutor(void); + +extern MemoryAccountIdType +MemoryAccounting_GetOrCreateNestedExecutorAccount(void); + +extern bool +MemoryAccounting_IsUnderNestedExecutor(void); + +extern bool +MemoryAccounting_IsMainExecutorCreated(void); + +/* + * MemoryAccounting_CreateExecutorMemoryAccount will create the main executor + * account. Any subsequent calls to this function will assign the executor to + * the 'X_NestedExecutor' account. + * + * If the explain_memory_verbosity guc is set to 'debug' or above, all + * executors will be given its own memory account. + */ +static inline MemoryAccountIdType +MemoryAccounting_CreateExecutorMemoryAccount() +{ + if (explain_memory_verbosity >= EXPLAIN_MEMORY_VERBOSITY_DEBUG) + return MemoryAccounting_CreateAccount(0, MEMORY_OWNER_TYPE_EXECUTOR); + else + if (MemoryAccounting_IsMainExecutorCreated()) + return MemoryAccounting_GetOrCreateNestedExecutorAccount(); + else + return MemoryAccounting_CreateMainExecutor(); +} + +/* + * MemoryAccounting_CreatePlanningMemoryAccount creates a memory account for + * query planning. If the planner is a direct child of the 'X_NestedExecutor' + * account, the planner account will also be assigned to 'X_NestedExecutor'. + * + * The planner account will always be created if the explain_memory_verbosity + * guc is set to 'debug' or above. + */ +extern MemoryAccountIdType +MemoryAccounting_CreatePlanningMemoryAccount(MemoryOwnerType type); + + #endif /* MEMACCOUNTING_H */ diff --git a/src/test/regress/expected/memconsumption.out b/src/test/regress/expected/memconsumption.out index e59f3fbc6a..e6c0abd26b 100644 --- a/src/test/regress/expected/memconsumption.out +++ b/src/test/regress/expected/memconsumption.out @@ -44,3 +44,146 @@ select sum_alien_consumption('SELECT t1.i, t2.j FROM test as t1 join test as t2 t (1 row) +create or replace function has_account_type(query text, search_text text) returns int as +$$ +import re +rv = plpy.execute('EXPLAIN ANALYZE '+ query) +comp_regex = re.compile("^\s+%s" % search_text) +count = 0 +for i in range(len(rv)): + cur_line = rv[i]['QUERY PLAN'] + m = comp_regex.match(cur_line) + if m is not None: + count = count + 1 +return count +$$ +language plpythonu; +-- Create functions that will generate nested SQL executors +CREATE OR REPLACE FUNCTION simple_plpgsql_function(int) RETURNS int AS $$ + DECLARE RESULT int; +BEGIN + SELECT count(*) FROM pg_class INTO RESULT; + RETURN RESULT + $1; +END; +$$ LANGUAGE plpgsql NO SQL; +CREATE OR REPLACE FUNCTION simple_sql_function(argument int) RETURNS bigint AS $$ +SELECT count(*) + argument FROM pg_class; +$$ LANGUAGE SQL STRICT VOLATILE; +-- Create a table with tuples only on one segement +CREATE TABLE all_tuples_on_seg0(i int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO all_tuples_on_seg0 VALUES (0), (0), (0); +SELECT gp_segment_id, count(*) FROM all_tuples_on_seg0 GROUP BY 1; + gp_segment_id | count +---------------+------- + 0 | 3 +(1 row) + +-- The X_NestedExecutor account is only created if we create an executor. +-- Because all the tuples in all_tuples_on_seg0 are on seg0, only seg0 should +-- create the X_NestedExecutor account. +set explain_memory_verbosity to detail; +-- We expect that only seg0 will create an X_NestedExecutor account, so this +-- should return '1' +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); + has_account_type +------------------ + 1 +(1 row) + +-- Each node will create a 'main' executor account, so we expect that this +-- will return '3', one per Query Executor. +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'Executor'); + has_account_type +------------------ + 3 +(1 row) + +-- Same as above, this should return '1' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); + has_account_type +------------------ + 1 +(1 row) + +-- Same as above, this should return '3' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'Executor'); + has_account_type +------------------ + 3 +(1 row) + +-- After setting explain_memory_verbosity to 'debug', the X_NestedExecutor +-- account will no longer be created. Instead, every time we evaluate a code +-- block in an sql or PL/pgSQL function, it will create a new executor account. +-- There are three tuples in all_tuples_on_seg0, so we should see three more +-- Executor accounts than when we had the explain_memory_verbosity guc set to +-- 'detail'. +set explain_memory_verbosity to debug; +-- We expect this will be '0' +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); + has_account_type +------------------ + 0 +(1 row) + +-- Because there are three tuples in all_tuples_on_seg0, we expect to see three +-- additional 'Executor' accounts created, a total number of '6'. +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'Executor'); + has_account_type +------------------ + 6 +(1 row) + +-- Expect '0' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); + has_account_type +------------------ + 0 +(1 row) + +-- Expect '6' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'Executor'); + has_account_type +------------------ + 6 +(1 row) + +-- Test X_NestedExecutor is created correctly inside multiple slice plans +set explain_memory_verbosity to detail; +-- We should see two TableScans- one per slice. Because only one segment has +-- tuples, only one segment per slice will create the 'X_NestedExecutor' +-- account. This will return '2'. +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'X_NestedExecutor'); + has_account_type +------------------ + 2 +(1 row) + +-- There will be two slices, and each slice will create an 'Executor' account +-- for a total of '6' 'Executor' accounts. +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'Executor'); + has_account_type +------------------ + 6 +(1 row) + +set explain_memory_verbosity to debug; +-- We don't create 'X_NestedExecutor' accounts when explain_memory_verbosity is +-- set to 'debug', so this will return '0' +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'X_NestedExecutor'); + has_account_type +------------------ + 0 +(1 row) + +-- Two slices, each returning three tuples. For each tuple we will create an +-- 'Executor' account. We also expect one main 'Executor' account per slice, so +-- expect '12' total Executor accounts +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'Executor'); + has_account_type +------------------ + 12 +(1 row) + diff --git a/src/test/regress/sql/memconsumption.sql b/src/test/regress/sql/memconsumption.sql index e30fad70e7..93179782ae 100644 --- a/src/test/regress/sql/memconsumption.sql +++ b/src/test/regress/sql/memconsumption.sql @@ -40,3 +40,89 @@ select sum_alien_consumption('SELECT t1.i, t2.j FROM test as t1 join test as t2 set execute_pruned_plan=off; select sum_alien_consumption('SELECT t1.i, t2.j FROM test as t1 join test as t2 on t1.i = t2.j') > 0; + +create or replace function has_account_type(query text, search_text text) returns int as +$$ +import re +rv = plpy.execute('EXPLAIN ANALYZE '+ query) +comp_regex = re.compile("^\s+%s" % search_text) +count = 0 +for i in range(len(rv)): + cur_line = rv[i]['QUERY PLAN'] + m = comp_regex.match(cur_line) + if m is not None: + count = count + 1 +return count +$$ +language plpythonu; + +-- Create functions that will generate nested SQL executors +CREATE OR REPLACE FUNCTION simple_plpgsql_function(int) RETURNS int AS $$ + DECLARE RESULT int; +BEGIN + SELECT count(*) FROM pg_class INTO RESULT; + RETURN RESULT + $1; +END; +$$ LANGUAGE plpgsql NO SQL; + + +CREATE OR REPLACE FUNCTION simple_sql_function(argument int) RETURNS bigint AS $$ +SELECT count(*) + argument FROM pg_class; +$$ LANGUAGE SQL STRICT VOLATILE; + +-- Create a table with tuples only on one segement +CREATE TABLE all_tuples_on_seg0(i int); +INSERT INTO all_tuples_on_seg0 VALUES (0), (0), (0); +SELECT gp_segment_id, count(*) FROM all_tuples_on_seg0 GROUP BY 1; + +-- The X_NestedExecutor account is only created if we create an executor. +-- Because all the tuples in all_tuples_on_seg0 are on seg0, only seg0 should +-- create the X_NestedExecutor account. +set explain_memory_verbosity to detail; +-- We expect that only seg0 will create an X_NestedExecutor account, so this +-- should return '1' +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); +-- Each node will create a 'main' executor account, so we expect that this +-- will return '3', one per Query Executor. +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'Executor'); +-- Same as above, this should return '1' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); +-- Same as above, this should return '3' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'Executor'); + +-- After setting explain_memory_verbosity to 'debug', the X_NestedExecutor +-- account will no longer be created. Instead, every time we evaluate a code +-- block in an sql or PL/pgSQL function, it will create a new executor account. +-- There are three tuples in all_tuples_on_seg0, so we should see three more +-- Executor accounts than when we had the explain_memory_verbosity guc set to +-- 'detail'. +set explain_memory_verbosity to debug; +-- We expect this will be '0' +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); +-- Because there are three tuples in all_tuples_on_seg0, we expect to see three +-- additional 'Executor' accounts created, a total number of '6'. +select has_account_type('select simple_sql_function(i) from all_tuples_on_seg0', 'Executor'); +-- Expect '0' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'X_NestedExecutor'); +-- Expect '6' +select has_account_type('select simple_plpgsql_function(i) from all_tuples_on_seg0', 'Executor'); + +-- Test X_NestedExecutor is created correctly inside multiple slice plans +set explain_memory_verbosity to detail; +-- We should see two TableScans- one per slice. Because only one segment has +-- tuples, only one segment per slice will create the 'X_NestedExecutor' +-- account. This will return '2'. +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'X_NestedExecutor'); +-- There will be two slices, and each slice will create an 'Executor' account +-- for a total of '6' 'Executor' accounts. +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'Executor'); + + +set explain_memory_verbosity to debug; +-- We don't create 'X_NestedExecutor' accounts when explain_memory_verbosity is +-- set to 'debug', so this will return '0' +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'X_NestedExecutor'); +-- Two slices, each returning three tuples. For each tuple we will create an +-- 'Executor' account. We also expect one main 'Executor' account per slice, so +-- expect '12' total Executor accounts +select has_account_type('select * from (select simple_sql_function(i) from all_tuples_on_seg0) a, (select simple_sql_function(i) from all_tuples_on_seg0) b', 'Executor'); -- GitLab