Add ICG tests for motion hazard deadlock case.

src/backend/cdb/cdbllize.c

@@ -1396,6 +1396,8 @@ motion_sanity_walker(Node *node, sanity_result_t *result)
 	 * We avoid this by pre-fetching all the inner tuples in such cases and
 	 * materializing them in some fashion, before moving on to outer_tuples.
 	 * This effectively breaks the cycle and prevents deadlock.
+	 *
+	 * Details: https://groups.google.com/a/greenplum.org/forum/#!msg/gpdb-dev/gMa1tW0x_fk/wuzvGXBaBAAJ
 	 */
 	switch (nodeTag(node))
 	{

src/backend/executor/nodeHashjoin.c

@@ -109,6 +109,8 @@ ExecHashJoin(HashJoinState *node)
 		 *
 		 * So now prefetch_inner is set (see createplan.c) if we have *any* motion
 		 * below us. If we don't have any motion, it doesn't matter.
+		 *
+		 * See motion_sanity_walker() for details on how a deadlock may occur.
 		 */
 		if (!node->prefetch_inner)
 		{

src/backend/executor/nodeMaterial.c

@@ -126,6 +126,8 @@ ExecMaterial(MaterialState *node)
 		 * and sending Motion operators to neutralize a deadlock hazard.
 		 * MPP TODO: Remove when a better solution is implemented.
 		 *
+		 * See motion_sanity_walker() for details on how a deadlock may occur.
+		 *
 		 * ShareInput: if the material node
 		 * is used to share input, we will need to fetch all rows and put
 		 * them in tuple store

src/backend/executor/nodeMergejoin.c

@@ -661,6 +661,8 @@ ExecMergeJoin(MergeJoinState *node)
 	 *
 	 * So now prefetch_inner is set (see createplan.c) if we have *any* motion
 	 * below us. If we don't have any motion, it doesn't matter.
+	 *
+	 * See motion_sanity_walker() for details on how a deadlock may occur.
 	 */
 	if (node->prefetch_inner)
 	{

src/backend/executor/nodeMotion.c

@@ -234,6 +234,7 @@ ExecMotion(MotionState * node)
 		{
 			if (node->ps.state->active_recv_id != motion->motionID)
 			{
+				/* See motion_sanity_walker() for details on how a deadlock may occur. */
 				elog(LOG, "DEADLOCK HAZARD: Updating active_motion_id from %d to %d",
 					 node->ps.state->active_recv_id, motion->motionID);
 				node->ps.state->active_recv_id = motion->motionID;

src/backend/executor/nodeNestloop.c

@@ -112,6 +112,8 @@ ExecNestLoop(NestLoopState *node)
 	 *
 	 * So now prefetch_inner is set (see createplan.c) if we have *any* motion
 	 * below us. If we don't have any motion, it doesn't matter.
+	 *
+	 * See motion_sanity_walker() for details on how a deadlock may occur.
 	 */
 	if (node->prefetch_inner)
 	{

src/backend/optimizer/plan/createplan.c

@@ -2980,6 +2980,8 @@ create_mergejoin_plan(PlannerInfo *root,
 	 * We need some kind of strict slackening operator (something which consumes all of its
 	 * input before producing a row of output) for our inner. And we need to prefetch that side
 	 * first.
+	 *
+	 * See motion_sanity_walker() for details on how a deadlock may occur.
 	 */
 	if (best_path->jpath.outerjoinpath->motionHazard && best_path->jpath.innerjoinpath->motionHazard)
 	{

src/test/regress/expected/deadlock.out

+-- 
+-- Test queries that can lead to deadlock. This is in a different file to
+-- isolate the test & quickly identify it if ICG doesn't complete for a very
+-- long time.
+--
+create schema deadlock;
+set search_path = deadlock;
+-- Test "Motion deadlock hazard" when Joins have Motions in both branches.
+-- see src/backend/cdb/cdbllize.c
+create table l (i int, j int) distributed by (i);
+create table r (i int, j int) distributed by (j);
+insert into l select i, 2 from generate_series(1, 100000) i;  -- one segment destination
+insert into r select i, 1 from generate_series(1, 100000) i;  -- one segment destination
+set enable_mergejoin = off;
+set enable_hashjoin = on;
+set enable_nestloop = off;
+-- Pick HJ
+select count(*) from gp_dist_random('l') left join gp_dist_random('r') on l.j + 1 = r.j;
+ count  
+--------
+ 100000
+(1 row)
+
+set enable_mergejoin = off;
+set enable_hashjoin = off;
+set enable_nestloop = on;
+-- Pick NLJ, gp_dist_random() forces MJ
+select count(*) from l left join r on l.j + 1 = r.j;
+ count  
+--------
+ 100000
+(1 row)
+
+set enable_mergejoin = on;
+set enable_hashjoin = off;
+set enable_nestloop = off;
+-- Pick MJ
+select count(*) from gp_dist_random('l') left outer join gp_dist_random('r') on l.j + 1 = r.j;
+ count  
+--------
+ 100000
+(1 row)
+
+drop schema if exists deadlock cascade;
+NOTICE:  drop cascades to table r
+NOTICE:  drop cascades to table l

src/test/regress/greenplum_schedule

@@ -29,6 +29,9 @@ test: indexjoin as_alias regex_gp gpparams with_clause transient_types gp_rules
 # dispatch should always run seperately from other cases.
 test: dispatch
 
+# deadlock tests run separately - because we don't know which one gets stuck.
+test: deadlock
+
 # This test will change the gp_workfile_limit_per_segment and will need to restart the DB.
 # It will also use faultinjector - so it needs to be in a group by itself.
 test: segspace_setup

src/test/regress/sql/deadlock.sql

+-- 
+-- Test queries that can lead to deadlock. This is in a different file to
+-- isolate the test & quickly identify it if ICG doesn't complete for a very
+-- long time.
+--
+
+
+create schema deadlock;
+set search_path = deadlock;
+
+-- Test "Motion deadlock hazard" when Joins have Motions in both branches.
+-- see src/backend/cdb/cdbllize.c
+
+create table l (i int, j int) distributed by (i);
+create table r (i int, j int) distributed by (j);
+
+insert into l select i, 2 from generate_series(1, 100000) i;  -- one segment destination
+insert into r select i, 1 from generate_series(1, 100000) i;  -- one segment destination
+
+set enable_mergejoin = off;
+set enable_hashjoin = on;
+set enable_nestloop = off;
+
+-- Pick HJ
+select count(*) from gp_dist_random('l') left join gp_dist_random('r') on l.j + 1 = r.j;
+
+
+set enable_mergejoin = off;
+set enable_hashjoin = off;
+set enable_nestloop = on;
+
+-- Pick NLJ, gp_dist_random() forces MJ
+select count(*) from l left join r on l.j + 1 = r.j;
+
+set enable_mergejoin = on;
+set enable_hashjoin = off;
+set enable_nestloop = off;
+
+-- Pick MJ
+select count(*) from gp_dist_random('l') left outer join gp_dist_random('r') on l.j + 1 = r.j;
+
+
+drop schema if exists deadlock cascade;