Add debug info for interconnect network timeout

It was very difficult to verify if interconnect is stucked in resending
phase or if there is udp resending latency within interconnect. To improve
it, this commit record a debug message every Gp_interconnect_debug_retry_interval
times when gp_log_interconnect is set to DEBUG.

src/backend/cdb/cdbvars.c

@@ -198,6 +198,7 @@ int			Gp_interconnect_min_rto = 20;
 int			Gp_interconnect_fc_method = INTERCONNECT_FC_METHOD_LOSS;
 int			Gp_interconnect_transmit_timeout = 3600;
 int			Gp_interconnect_min_retries_before_timeout = 100;
+int			Gp_interconnect_debug_retry_interval= 10;
 
 int			Gp_interconnect_hash_multiplier = 2;		/* sets the size of the
 														 * hash table used by

src/backend/cdb/motion/ic_udpifc.c

@@ -4899,6 +4899,14 @@ checkNetworkTimeout(ICBuffer *buf, uint64 now)
 	 * by OS for a long time. In this case, only a few times are tried.
 	 * Thus, the GUC Gp_interconnect_min_retries_before_timeout is added here.
 	 */
+	if (gp_log_interconnect >= GPVARS_VERBOSITY_DEBUG &&
+	   buf->nRetry % Gp_interconnect_debug_retry_interval == 0)
+	{
+		ereport(LOG, (errmsg("resending packet (seq %d) to %s (pid %d cid %d) with %d retries in %lu seconds",
+							 buf->pkt->seq, buf->conn->remoteHostAndPort, buf->pkt->dstPid,
+							 buf->pkt->dstContentId, buf->nRetry, (now - buf->sentTime) / 1000 / 1000 )));
+	}
+
 	if ((buf->nRetry > Gp_interconnect_min_retries_before_timeout) && (now - buf->sentTime) > ((uint64)Gp_interconnect_transmit_timeout * 1000 * 1000))
 	{
 		ereport(ERROR, (errcode(ERRCODE_GP_INTERCONNECTION_ERROR),

src/backend/utils/misc/guc_gp.c

@@ -3831,6 +3831,16 @@ struct config_int ConfigureNamesInt_gp[] =
 		100, 1, 4096, NULL, NULL
 	},
 
+	{
+		{"gp_interconnect_debug_retry_interval", PGC_USERSET, GP_ARRAY_TUNING,
+			gettext_noop("Sets the interval by retry times to record a debug message for retry."),
+			NULL,
+			GUC_GPDB_ADDOPT
+		},
+		&Gp_interconnect_debug_retry_interval,
+		10, 1, 4096, NULL, NULL
+	},
+
 	{
 		{"gp_udp_bufsize_k", PGC_BACKEND, GP_ARRAY_TUNING,
 			gettext_noop("Sets recv buf size of UDP interconnect, for testing."),

src/include/cdb/cdbvars.h

@@ -439,6 +439,7 @@ extern int	Gp_interconnect_default_rtt;
 extern int	Gp_interconnect_min_rto;
 extern int  Gp_interconnect_transmit_timeout;
 extern int	Gp_interconnect_min_retries_before_timeout;
+extern int	Gp_interconnect_debug_retry_interval;
 
 /* UDP recv buf size in KB.  For testing */
 extern int 	Gp_udp_bufsize_k;