From e580dab88352542b8bec0dd8d10d189b38ae2563 Mon Sep 17 00:00:00 2001
From: xiong-gang <gxiong@pivotal.io>
Date: Thu, 4 Jul 2019 10:57:45 +0800
Subject: [PATCH] Change the size of interconnect connection hash table

As discussed in #6932, the size of interconnect connection hash table is 16,
which would be too small for large cluster and could bring too many hash table
collisions. Set to (2 * number segments).
---
 src/backend/cdb/dispatcher/cdbgang.c       | 19 +++++++++++++++----
 src/backend/cdb/dispatcher/cdbgang_async.c |  6 +++++-
 src/backend/cdb/motion/ic_udpifc.c         |  4 ++--
 src/include/cdb/cdbgang.h                  |  3 ++-
 4 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/backend/cdb/dispatcher/cdbgang.c b/src/backend/cdb/dispatcher/cdbgang.c
index 06d5841210..de07c27c0c 100644
--- a/src/backend/cdb/dispatcher/cdbgang.c
+++ b/src/backend/cdb/dispatcher/cdbgang.c
@@ -60,6 +60,12 @@ int			qe_identifier = 0;
  */
 int			host_segments = 0;
 
+/*
+ * size of hash table of interconnect connections
+ * equals to 2 * (the number of total segments)
+ */
+int			ic_htab_size = 0;
+
 Gang      *CurrentGangCreating = NULL;
 
 CreateGangFunc pCreateGangFunc = cdbgang_createGang_async;
@@ -357,7 +363,7 @@ makeOptions(void)
  */
 bool
 build_gpqeid_param(char *buf, int bufsz,
-				   bool is_writer, int identifier, int hostSegs)
+				   bool is_writer, int identifier, int hostSegs, int icHtabSize)
 {
 	int		len;
 #ifdef HAVE_INT64_TIMESTAMP
@@ -370,9 +376,9 @@ build_gpqeid_param(char *buf, int bufsz,
 #endif
 #endif
 
-	len = snprintf(buf, bufsz, "%d;" TIMESTAMP_FORMAT ";%s;%d;%d",
+	len = snprintf(buf, bufsz, "%d;" TIMESTAMP_FORMAT ";%s;%d;%d;%d",
 				   gp_session_id, PgStartTime,
-				   (is_writer ? "true" : "false"), identifier, hostSegs);
+				   (is_writer ? "true" : "false"), identifier, hostSegs, icHtabSize);
 
 	return (len > 0 && len < bufsz);
 }
@@ -444,11 +450,16 @@ cdbgang_parse_gpqeid_params(struct Port *port __attribute__((unused)),
 		host_segments = (int) strtol(cp, NULL, 10);
 	}
 
+	if (gpqeid_next_param(&cp, &np))
+	{
+		ic_htab_size = (int) strtol(cp, NULL, 10);
+	}
+
 	/* Too few items, or too many? */
 	if (!cp || np)
 		goto bad;
 
-	if (gp_session_id <= 0 || PgStartTime <= 0 || qe_identifier < 0 || host_segments <= 0)
+	if (gp_session_id <= 0 || PgStartTime <= 0 || qe_identifier < 0 || host_segments <= 0 || ic_htab_size <= 0)
 		goto bad;
 
 	pfree(gpqeid);
diff --git a/src/backend/cdb/dispatcher/cdbgang_async.c b/src/backend/cdb/dispatcher/cdbgang_async.c
index 9628fe52e8..4278ea7761 100644
--- a/src/backend/cdb/dispatcher/cdbgang_async.c
+++ b/src/backend/cdb/dispatcher/cdbgang_async.c
@@ -54,6 +54,7 @@ cdbgang_createGang_async(List *segments, SegmentType segmentType)
 	int		i = 0;
 	int		size = 0;
 	bool	retry = false;
+	int		totalSegs = 0;
 
 	/*
 	 * true means connection status is confirmed, either established or in
@@ -72,6 +73,8 @@ cdbgang_createGang_async(List *segments, SegmentType segmentType)
 	/* allocate and initialize a gang structure */
 	newGangDefinition = buildGangDefinition(segments, segmentType);
 	CurrentGangCreating = newGangDefinition;
+	totalSegs = getgpsegmentCount();
+	Assert(totalSegs > 0);
 
 create_gang_retry:
 	Assert(newGangDefinition != NULL);
@@ -120,7 +123,8 @@ create_gang_retry:
 			ret = build_gpqeid_param(gpqeid, sizeof(gpqeid),
 									 segdbDesc->isWriter,
 									 segdbDesc->identifier,
-									 segdbDesc->segment_database_info->hostSegs);
+									 segdbDesc->segment_database_info->hostSegs,
+									 totalSegs * 2);
 
 			if (!ret)
 				ereport(ERROR,
diff --git a/src/backend/cdb/motion/ic_udpifc.c b/src/backend/cdb/motion/ic_udpifc.c
index 6f62412269..d757e973f1 100644
--- a/src/backend/cdb/motion/ic_udpifc.c
+++ b/src/backend/cdb/motion/ic_udpifc.c
@@ -177,7 +177,6 @@ struct ConnHashTable
 	int			size;
 };
 
-#define DEFAULT_CONN_HTAB_SIZE 16
 #define CONN_HASH_VALUE(icpkt) ((uint32)((((icpkt)->srcPid ^ (icpkt)->dstPid)) + (icpkt)->dstContentId))
 #define CONN_HASH_MATCH(a, b) (((a)->motNodeId == (b)->motNodeId && \
 								(a)->dstContentId == (b)->dstContentId && \
@@ -1511,7 +1510,8 @@ initConnHashTable(ConnHashTable *ht, MemoryContext cxt)
 	int			i;
 
 	ht->cxt = cxt;
-	ht->size = DEFAULT_CONN_HTAB_SIZE;
+	ht->size = Gp_role == GP_ROLE_DISPATCH ? (getgpsegmentCount() * 2) : ic_htab_size;
+	Assert(ht->size > 0);
 
 	if (ht->cxt)
 	{
diff --git a/src/include/cdb/cdbgang.h b/src/include/cdb/cdbgang.h
index b43e6de130..c3200dcc6d 100644
--- a/src/include/cdb/cdbgang.h
+++ b/src/include/cdb/cdbgang.h
@@ -53,6 +53,7 @@ typedef struct Gang
 extern int qe_identifier;
 
 extern int host_segments;
+extern int ic_htab_size;
 
 extern MemoryContext GangContext;
 extern Gang *CurrentGangCreating;
@@ -86,7 +87,7 @@ extern void CheckForResetSession(void);
 extern struct SegmentDatabaseDescriptor *getSegmentDescriptorFromGang(const Gang *gp, int seg);
 
 Gang *buildGangDefinition(List *segments, SegmentType segmentType);
-bool build_gpqeid_param(char *buf, int bufsz, bool is_writer, int identifier, int hostSegs);
+bool build_gpqeid_param(char *buf, int bufsz, bool is_writer, int identifier, int hostSegs, int icHtabSize);
 
 char *makeOptions(void);
 extern bool segment_failure_due_to_recovery(const char *error_message);
-- 
GitLab