Add a long living account for Relinquished Memory

There are cases where during execution a Memory Intensive Operator (MI)
may not use all the memory that is allocated to it. This means that this
extra memory (quota - allocated) can be relinquished for other MI nodes
to use during execution of a statement. For example

->  Hash Join
         ->  HashAggregate
         ->  Hash
In the above query fragment the HashJoin operator has a MI operator for
both its inner and outer subtree. If there ever is the case that the
Hash node used much less memory than was given as its quota it will now
call MemoryAccounting_DeclareDone() and the difference between its
quota and allocated amount will be added to the allocated amount of the
RelinquishedPool. Doing this will enable HashAggregate to request memory
from this RelinquishedPool if it exhausts its quota to prevent spilling.

This PR adds two new API's to the MemoryAccounting Framework

MemoryAccounting_DeclareDone(): Add the difference between a memory
account's quota and its allocated amount to the long living
RelinquishedPool

MemoryAccounting_RequestQuotaIncrease(): Retrieve all relinquished
memory by incrementing an operator's operatorMemKb and setting the
RelinquishedPool to 0

Note: This PR introduces the facility for Hash to relinquish memory to
the RelinquishedPool memory account and for the Agg operator
(specifically HashAgg) to request an increase to its quota before it
builds its hash table. This commit does not generally apply this
paradigm to all MI operators
Signed-off-by: NSambitesh Dash <sdash@pivotal.io>
Signed-off-by: NMelanie Plageman <mplageman@pivotal.io>

src/backend/executor/execUtils.c

@@ -74,6 +74,8 @@
 #include "cdb/cdbllize.h"
 #include "utils/workfile_mgr.h"
 
+#include "cdb/memquota.h"
+
 static void ShutdownExprContext(ExprContext *econtext);
 
 
@@ -2170,7 +2172,12 @@ uint64 PlanStateOperatorMemKB(const PlanState *ps)
 	}
 	else
 	{
-		result = ps->plan->operatorMemKB;
+		if (IsA(ps, AggState))
+		{
+			result = ps->plan->operatorMemKB + MemoryAccounting_RequestQuotaIncrease();
+		}
+		else
+			result = ps->plan->operatorMemKB;
 	}
 	
 	return result;

src/backend/executor/nodeHash.c

@@ -147,6 +147,7 @@ MultiExecHash(HashState *node)
 			}
 		}
 	}
+	MemoryAccounting_DeclareDone();
 
 	/* Now we have set up all the initial batches & primary overflow batches. */
 	hashtable->nbatch_outstart = hashtable->nbatch;

src/backend/executor/nodeHashjoin.c

@@ -1205,7 +1205,7 @@ ReleaseHashTable(HashJoinState *node)
 {
 	if (node->hj_HashTable)
 	{
-		HashState  *hashState = (HashState *) innerPlanState(node);
+		HashState *hashState = (HashState *) innerPlanState(node);
 
 		/* This hashtable should not have been released already! */
 		Assert(!node->hj_HashTable->eagerlyReleased);

src/backend/utils/mmgr/memaccounting.c

@@ -178,6 +178,12 @@ MemoryAccount *RolloverMemoryAccount = NULL;
  */
 MemoryAccount *AlienExecutorMemoryAccount = NULL;
 
+/*
+ * RelinqishedPoolMemoryAccount is a shared executor account that tracks amount
+ * of additional free memory available for execution
+ */
+MemoryAccount *RelinquishedPoolMemoryAccount = NULL;
+
 /*
  * Total outstanding (i.e., allocated - freed) memory across all
  * memory accounts, including RolloverMemoryAccount
@@ -230,6 +236,40 @@ MemoryAccounting_Reset()
 	InitMemoryAccounting();
 }
 
+/*
+ * MemoryAccounting_DeclareDone
+ * 		Increments the RelinquishedPoolMemoryAccount by the difference between the current
+ * 		Memory Account's quota and allocated amount.
+ * 		This should only be called when a MemoryAccount is certain that it will not
+ * 		allocate any more memory
+ */
+uint64
+MemoryAccounting_DeclareDone()
+{
+	MemoryAccount *currentAccount = MemoryAccounting_ConvertIdToAccount(ActiveMemoryAccountId);
+	uint64 relinquished = 0;
+	if (currentAccount->maxLimit > 0 && currentAccount->maxLimit > currentAccount->allocated)
+	{
+		relinquished = currentAccount->maxLimit - currentAccount->allocated;
+		RelinquishedPoolMemoryAccount->allocated += relinquished;
+		currentAccount->relinquishedMemory = relinquished;
+	}
+
+	elog(DEBUG2, "Memory Account %d relinquished %u bytes of memory", currentAccount->ownerType, relinquished);
+	return relinquished;
+}
+
+uint64
+MemoryAccounting_RequestQuotaIncrease()
+{
+	MemoryAccount *currentAccount = MemoryAccounting_ConvertIdToAccount(ActiveMemoryAccountId);
+
+	uint64 result = RelinquishedPoolMemoryAccount->allocated;
+	currentAccount->acquiredMemory = result;
+	RelinquishedPoolMemoryAccount->allocated = 0;
+	return result;
+}
+
 /*
  * MemoryAccounting_CreateAccount
  *		Public method to create a memory account. We use this to force outside
@@ -547,6 +587,9 @@ InitLongLivingAccounts() {
 			longLivingMemoryAccountArray[MEMORY_OWNER_TYPE_MemAccount];
 	AlienExecutorMemoryAccount =
 			longLivingMemoryAccountArray[MEMORY_OWNER_TYPE_Exec_AlienShared];
+	RelinquishedPoolMemoryAccount =
+			longLivingMemoryAccountArray[MEMORY_OWNER_TYPE_Exec_RelinquishedPool];
+
 }
 
 /* Initializes all the short living accounts */
@@ -648,6 +691,8 @@ InitializeMemoryAccount(MemoryAccount *newAccount, long maxLimit, MemoryOwnerTyp
 	}
 	newAccount->freed = 0;
 	newAccount->peak = 0;
+	newAccount->relinquishedMemory = 0;
+	newAccount->acquiredMemory = 0;
 	newAccount->parentId = parentAccountId;
 
 	if (ownerType <= MEMORY_OWNER_TYPE_END_LONG_LIVING)
@@ -691,6 +736,7 @@ CreateMemoryAccountImpl(long maxLimit, MemoryOwnerType ownerType, MemoryAccountI
 			ownerType == MEMORY_OWNER_TYPE_Rollover ||
 			ownerType == MEMORY_OWNER_TYPE_MemAccount ||
 			ownerType == MEMORY_OWNER_TYPE_Exec_AlienShared ||
+			ownerType == MEMORY_OWNER_TYPE_Exec_RelinquishedPool ||
 			(MemoryAccountMemoryContext != NULL && MemoryAccountMemoryAccount != NULL));
 
 	if (ownerType <= MEMORY_OWNER_TYPE_END_LONG_LIVING || ownerType == MEMORY_OWNER_TYPE_Top)
@@ -954,15 +1000,22 @@ MemoryAccountToString(MemoryAccountTree *memoryAccountTreeNode, void *context, u
 
 	MemoryAccount *memoryAccount = memoryAccountTreeNode->account;
 
+	if (memoryAccount->ownerType == MEMORY_OWNER_TYPE_Exec_RelinquishedPool) return CdbVisit_Walk;
+
 	MemoryAccountSerializerCxt *memAccountCxt = (MemoryAccountSerializerCxt*) context;
 
 	appendStringInfoFill(memAccountCxt->buffer, 2 * depth, ' ');
 
 	Assert(memoryAccount->peak >= MemoryAccounting_GetBalance(memoryAccount));
 	/* We print only integer valued memory consumption, in standard GPDB KB unit */
-	appendStringInfo(memAccountCxt->buffer, "%s: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: " UINT64_FORMAT "bytes.\n",
+	appendStringInfo(memAccountCxt->buffer, "%s: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: " UINT64_FORMAT " bytes.",
 			MemoryAccounting_GetOwnerName(memoryAccount->ownerType),
 			memoryAccount->peak, MemoryAccounting_GetBalance(memoryAccount), memoryAccount->maxLimit);
+	if (memoryAccount->relinquishedMemory > 0)
+		appendStringInfo(memAccountCxt->buffer, " Relinquished Memory: " UINT64_FORMAT " bytes. ", memoryAccount->relinquishedMemory);
+	if (memoryAccount->acquiredMemory > 0)
+		appendStringInfo(memAccountCxt->buffer, " Acquired Additional Memory: " UINT64_FORMAT " bytes.", memoryAccount->acquiredMemory);
+	appendStringInfo(memAccountCxt->buffer, "\n");
 
 	memAccountCxt->memoryAccountCount++;
 
@@ -989,6 +1042,8 @@ MemoryAccounting_GetOwnerName(MemoryOwnerType ownerType)
 		return "MemAcc";
 	case MEMORY_OWNER_TYPE_Exec_AlienShared:
 		return "X_Alien";
+	case MEMORY_OWNER_TYPE_Exec_RelinquishedPool:
+		return "RelinquishedPool";
 
 		/* Short living accounts */
 	case MEMORY_OWNER_TYPE_Top:
@@ -1334,6 +1389,13 @@ AdvanceMemoryAccountingGeneration()
 	 */
 	RolloverMemoryAccount->peak = Max(RolloverMemoryAccount->peak, MemoryAccountingPeakBalance);
 
+	/*
+	 * Reset the RelinquishedPool Long living account, the amount should not be carried between memory account generations
+	 */
+	RelinquishedPoolMemoryAccount->allocated = 0;
+	RelinquishedPoolMemoryAccount->peak = 0;
+	RelinquishedPoolMemoryAccount->freed = 0;
+
 	liveAccountStartId = nextAccountId;
 
 	Assert(RolloverMemoryAccount->peak >= MemoryAccountingPeakBalance);

src/backend/utils/mmgr/test/memaccounting_test.c

@@ -897,15 +897,15 @@ test__MemoryAccounting_CombinedAccountArrayToString__Validate(void **state)
 
 	uint32 totalSerialized = MemoryAccounting_Serialize(&serializedBytes);
 
-	char *templateString = "Root: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  Top: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\
-    X_Hash: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-      X_Sort: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-      X_SeqScan: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  X_Alien: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  MemAcc: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\
-  Rollover: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  SharedHeader: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n";
+	char *templateString = "Root: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  Top: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\
+    X_Hash: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+      X_Sort: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+      X_SeqScan: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  X_Alien: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  MemAcc: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\
+  Rollover: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  SharedHeader: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n";
 
 	char		buf[MAX_OUTPUT_BUFFER_SIZE];
 
@@ -930,7 +930,7 @@ test__MemoryAccounting_CombinedAccountArrayToString__Validate(void **state)
 void
 test__MemoryAccounting_GetAccountName__Validate(void **state)
 {
-	char* longLivingNames[] = {"Root", "SharedHeader", "Rollover", "MemAcc", "X_Alien"};
+	char* longLivingNames[] = {"Root", "SharedHeader", "Rollover", "MemAcc", "X_Alien", "RelinquishedPool"};
 
 	char* shortLivingNames[] = {"Top", "Main", "Parser", "Planner", "PlannerHook", "Optimizer", "Dispatcher", "Serializer", "Deserializer",
 			"Executor", "X_Result", "X_Append", "X_Sequence", "X_BitmapAnd", "X_BitmapOr", "X_SeqScan", "X_ExternalScan",
@@ -1028,14 +1028,14 @@ void
 test__MemoryAccounting_ToString__Validate(void **state)
 {
 	char *templateString =
-"Root: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  Top: Peak/Cur %" PRIu64 "/%" PRIu64 "bytes. Quota: 0bytes.\n\
-    X_Agg: Peak/Cur %" PRIu64 "/%" PRIu64 "bytes. Quota: 0bytes.\n\
-  X_Alien: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  MemAcc: Peak/Cur %" PRIu64 "/%" PRIu64 "bytes. Quota: 0bytes.\n\
-  Rollover: Peak/Cur %" PRIu64 "/%" PRIu64 "bytes. Quota: 0bytes.\n\
-    X_Hash: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  SharedHeader: Peak/Cur %" PRIu64 "/%" PRIu64 "bytes. Quota: 0bytes.\n";
+"Root: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  Top: Peak/Cur %" PRIu64 "/%" PRIu64 " bytes. Quota: 0 bytes.\n\
+    X_Agg: Peak/Cur %" PRIu64 "/%" PRIu64 " bytes. Quota: 0 bytes.\n\
+  X_Alien: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  MemAcc: Peak/Cur %" PRIu64 "/%" PRIu64 " bytes. Quota: 0 bytes.\n\
+  Rollover: Peak/Cur %" PRIu64 "/%" PRIu64 " bytes. Quota: 0 bytes.\n\
+    X_Hash: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  SharedHeader: Peak/Cur %" PRIu64 "/%" PRIu64 " bytes. Quota: 0 bytes.\n";
 
 	MemoryAccountIdType oldAccountId = CreateMemoryAccountImpl(0, MEMORY_OWNER_TYPE_Exec_Hash, ActiveMemoryAccountId);
 	/* Make oldAccountId obsolete */
@@ -1105,8 +1105,9 @@ memory: SharedHeader, 2, 1, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64
 memory: Rollover, 3, 1, 0, 0, 0, 0, 0\n\
 memory: MemAcc, 4, 1, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "\n\
 memory: X_Alien, 5, 1, 0, 0, 0, 0, 0\n\
-memory: Top, 6, 1, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "\n\
-memory: X_Hash, 7, 6, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "\n";
+memory: RelinquishedPool, 6, 1, 0, 0, 0, 0, 0\n\
+memory: Top, 7, 1, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "\n\
+memory: X_Hash, 8, 7, 0, %" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "\n";
 
 	/* ActiveMemoryAccount should be Top at this point */
 	MemoryAccount *newAccount = MemoryAccounting_ConvertIdToAccount(
@@ -1156,13 +1157,13 @@ void
 test__MemoryAccounting_PrettyPrint__GeneratesCorrectString(void **state)
 {
 
-	char *templateString = "Root: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  Top: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\
-    X_Hash: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\
-  X_Alien: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  MemAcc: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\
-  Rollover: Peak/Cur 0/0bytes. Quota: 0bytes.\n\
-  SharedHeader: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT "bytes. Quota: 0bytes.\n\n";
+	char *templateString = "Root: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  Top: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\
+    X_Hash: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\
+  X_Alien: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  MemAcc: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\
+  Rollover: Peak/Cur 0/0 bytes. Quota: 0 bytes.\n\
+  SharedHeader: Peak/Cur " UINT64_FORMAT "/" UINT64_FORMAT " bytes. Quota: 0 bytes.\n\n";
 
 	/* ActiveMemoryAccount should be Top at this point */
 	MemoryAccount *newAccount = MemoryAccounting_ConvertIdToAccount(
@@ -1244,9 +1245,10 @@ test__MemoryAccounting_SaveToFile__GeneratesCorrectString(void **state)
 	int lineNo = 0;
 
 	int memoryOwnerTypes[] = {MEMORY_STAT_TYPE_VMEM_RESERVED, MEMORY_STAT_TYPE_MEMORY_ACCOUNTING_PEAK,
-			MEMORY_OWNER_TYPE_LogicalRoot, MEMORY_OWNER_TYPE_Top, MEMORY_OWNER_TYPE_Exec_Hash ,
-			MEMORY_OWNER_TYPE_Exec_AlienShared, MEMORY_OWNER_TYPE_MemAccount,
-			MEMORY_OWNER_TYPE_Rollover,
+			MEMORY_OWNER_TYPE_LogicalRoot, MEMORY_OWNER_TYPE_Top, MEMORY_OWNER_TYPE_Exec_Hash,
+			MEMORY_OWNER_TYPE_Exec_RelinquishedPool,
+			MEMORY_OWNER_TYPE_Exec_AlienShared,
+			MEMORY_OWNER_TYPE_MemAccount, MEMORY_OWNER_TYPE_Rollover,
 			MEMORY_OWNER_TYPE_SharedChunkHeader};
 
 	char runId[80];
@@ -1305,6 +1307,11 @@ test__MemoryAccounting_SaveToFile__GeneratesCorrectString(void **state)
 			/* Verify allocated and peak, but don't verify freed, as freed will be after MemoryAccounting_SaveToFile is finished */
 			assert_true(peak == newAccount->peak && allocated == newAccount->allocated);
 		}
+		else if (ownerType == MEMORY_OWNER_TYPE_Exec_RelinquishedPool)
+		{
+			assert_true(peak == RelinquishedPoolMemoryAccount->peak &&
+					allocated == RelinquishedPoolMemoryAccount->allocated && freed == RelinquishedPoolMemoryAccount->freed);
+		}
 		else if (ownerType == MEMORY_OWNER_TYPE_Exec_AlienShared)
 		{
 			assert_true(peak == AlienExecutorMemoryAccount->peak &&

src/backend/utils/resource_manager/memquota.c

@@ -56,7 +56,6 @@ typedef struct PolicyAutoContext
 static bool PolicyAutoPrelimWalker(Node *node, PolicyAutoContext *context);
 static bool	PolicyAutoAssignWalker(Node *node, PolicyAutoContext *context);
 static bool IsAggMemoryIntensive(Agg *agg);
-static bool IsMemoryIntensiveOperator(Node *node, PlannedStmt *stmt);
 
 struct OperatorGroupNode;
 
@@ -234,7 +233,7 @@ IsResultMemoryIntesive(Result *res)
 /**
  * Is an operator memory intensive?
  */
-static bool
+bool
 IsMemoryIntensiveOperator(Node *node, PlannedStmt *stmt)
 {
 	Assert(is_plan_node(node));

src/include/cdb/memquota.h

@@ -57,6 +57,11 @@ extern uint64 PolicyAutoStatementMemForNoSpillKB(PlannedStmt *stmt, uint64 minOp
  */
 extern bool IsResultMemoryIntesive(Result *res);
 
+/**
+ * Is operator memory intensive
+ */
+extern bool IsMemoryIntensiveOperator(Node *node, PlannedStmt *stmt);
+
 /*
  * Calculate the amount of memory reserved for the query
  */

src/include/utils/memaccounting.h

@@ -76,7 +76,8 @@ typedef enum MemoryOwnerType
 	MEMORY_OWNER_TYPE_Rollover,
 	MEMORY_OWNER_TYPE_MemAccount,
 	MEMORY_OWNER_TYPE_Exec_AlienShared,
-	MEMORY_OWNER_TYPE_END_LONG_LIVING = MEMORY_OWNER_TYPE_Exec_AlienShared,
+	MEMORY_OWNER_TYPE_Exec_RelinquishedPool,
+	MEMORY_OWNER_TYPE_END_LONG_LIVING = MEMORY_OWNER_TYPE_Exec_RelinquishedPool,
 	/* End of long-living accounts */
 
 	/* Short-living accounts */
@@ -233,4 +234,10 @@ MemoryAccounting_SaveToLog(void);
 extern void
 MemoryAccounting_PrettyPrint(void);
 
+extern uint64
+MemoryAccounting_DeclareDone();
+
+extern uint64
+MemoryAccounting_RequestQuotaIncrease();
+
 #endif   /* MEMACCOUNTING_H */

src/include/utils/memaccounting_private.h

@@ -35,6 +35,16 @@ typedef struct MemoryAccount {
 	 */
 	uint64 maxLimit;
 
+	/*
+	 * Amount of memory relinquished
+	 */
+	uint64 relinquishedMemory;
+
+	/*
+	 * Amount of memory acquired from relinquished pool
+	 */
+	uint64 acquiredMemory;
+
 	MemoryAccountIdType id;
 	MemoryAccountIdType parentId;
 } MemoryAccount;