Fixup on splitJoinQualExpr().

For LASJ join, the result is supposed to be empty if there is NULL in
the inner side. To check for the NULLness, the join clauses are split
into outer and inner argument values so that we can evaluate those
subexpressions separately.

This patch adds verification when doing extraction that the join clauses
are in the format of 'foo = ANY bar' and that the equality operation is
strict.

This patch fixes issue #6389, in which the equality operator is
implemented by a function. In this case, the length of arguments is one.
So when it tries to extract the second argument, it refers to an invalid
pointer and gets segfaulted.
Reviewed-by: NEkta Khanna <ekhanna@pivotal.io>
Reviewed-by: NHeikki Linnakangas <hlinnakangas@pivotal.io>
Reviewed-by: NMelanie Plageman <mplageman@pivotal.io>

src/backend/executor/nodeNestloop.c

@@ -25,6 +25,8 @@
 
 #include "executor/execdebug.h"
 #include "executor/nodeNestloop.h"
+#include "optimizer/clauses.h"
+#include "utils/lsyscache.h"
 #include "utils/memutils.h"
 
 static void splitJoinQualExpr(NestLoopState *nlstate);
@@ -634,11 +636,31 @@ splitJoinQualExpr(NestLoopState *nlstate)
  * given lists:
  *   - lclauses for the left side of the expression,
  *   - rclauses for the right side
+ *
+ * This function is only used for LASJ. Once we find a NULL from inner side, we
+ * can skip the join and just return an empty set as result. This is only true
+ * if the equality operator is strict, that is, if a tuple from inner side is
+ * NULL then the equality operator returns NULL.
+ *
+ * If the number of arguments is not two, we just return leaving lclauses and
+ * rclauses remaining NULL. In this case, the LASJ join would be actually
+ * performed.
  * ----------------------------------------------------------------
  */
 static void
 extractFuncExprArgs(FuncExprState *fstate, List **lclauses, List **rclauses)
 {
-	*lclauses = lappend(*lclauses, linitial(fstate->args));
-	*rclauses = lappend(*rclauses, lsecond(fstate->args));
+	Node *clause;
+
+	if (list_length(fstate->args) != 2)
+		return;
+
+	/* Check for strictness of the equality operator */
+	clause = (Node *)fstate->xprstate.expr;
+	if ((is_opclause(clause) && op_strict(((OpExpr *) clause)->opno)) ||
+			(is_funcclause(clause) && func_strict(((FuncExpr *) clause)->funcid)))
+	{
+		*lclauses = lappend(*lclauses, linitial(fstate->args));
+		*rclauses = lappend(*rclauses, lsecond(fstate->args));
+	}
 }

src/test/regress/expected/notin.out

@@ -1137,9 +1137,38 @@ select c1 from t1 where c1 not in (select c2 from t2 where c2 > 4) and c1 > 2;
   9
 (7 rows)
 
+-- Test if the equality operator is implemented by a SQL function
+--
+--q45
+--
+create domain absint as int4;
+create function iszero(absint) returns bool as $$ begin return $1::int4 = 0; end; $$ language plpgsql immutable strict;
+create or replace function abseq (absint, absint) returns bool as $$ select iszero(abs($1) - abs($2)); $$ language sql immutable strict;
+create operator = (PROCEDURE = abseq, leftarg=absint, rightarg=absint);
+explain select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+                                                  QUERY PLAN                                                  
+--------------------------------------------------------------------------------------------------------------
+ Gather Motion 3:1  (slice2; segments: 3)  (cost=10000000000.00..10000000062.89 rows=4 width=4)
+   ->  Nested Loop Left Anti Semi (Not-In) Join  (cost=10000000000.00..10000000062.89 rows=2 width=4)
+         Join Filter: iszero(((abs(((t1.c1)::absint)::integer) - abs(((t1n.c1n)::absint)::integer)))::absint)
+         ->  Seq Scan on t1  (cost=0.00..3.10 rows=4 width=4)
+         ->  Materialize  (cost=0.00..3.45 rows=7 width=4)
+               ->  Broadcast Motion 3:3  (slice1; segments: 3)  (cost=0.00..3.35 rows=7 width=4)
+                     ->  Seq Scan on t1n  (cost=0.00..3.07 rows=3 width=4)
+ Planning time: 0.865 ms
+ Optimizer: legacy query optimizer
+(9 rows)
+
+select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+ c1 
+----
+(0 rows)
+
 reset search_path;
 drop schema notin cascade;
-NOTICE:  drop cascades to 7 other objects
+NOTICE:  drop cascades to 11 other objects
 DETAIL:  drop cascades to table notin.t1
 drop cascades to table notin.t2
 drop cascades to table notin.t3
@@ -1147,3 +1176,7 @@ drop cascades to table notin.t4
 drop cascades to table notin.t1n
 drop cascades to table notin.g1
 drop cascades to table notin.l1
+drop cascades to type notin.absint
+drop cascades to function notin.iszero(notin.absint)
+drop cascades to function notin.abseq(notin.absint,notin.absint)
+drop cascades to operator notin.=(notin.absint,notin.absint)

src/test/regress/expected/notin_optimizer.out

@@ -1201,9 +1201,51 @@ select c1 from t1 where c1 not in (select c2 from t2 where c2 > 4) and c1 > 2;
   7
 (7 rows)
 
+-- Test if the equality operator is implemented by a SQL function
+--
+--q45
+--
+create domain absint as int4;
+create function iszero(absint) returns bool as $$ begin return $1::int4 = 0; end; $$ language plpgsql immutable strict;
+create or replace function abseq (absint, absint) returns bool as $$ select iszero(abs($1) - abs($2)); $$ language sql immutable strict;
+create operator = (PROCEDURE = abseq, leftarg=absint, rightarg=absint);
+explain select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+                                                                                                                                                                               QUERY PLAN                                                                                                                                                                                
+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ Gather Motion 3:1  (slice3; segments: 3)  (cost=0.00..1324036.49 rows=10 width=4)
+   ->  Result  (cost=0.00..1324036.49 rows=4 width=4)
+         Filter: (NOT CASE WHEN (NOT ((t1.c1)::absint IS NULL)) THEN CASE WHEN ((pg_catalog.sum((sum((CASE WHEN (((t1n.c1n)::absint) IS NULL) THEN 1 ELSE 0 END))))) = (count((count())))) THEN NULL::boolean WHEN (NOT ((pg_catalog.sum((sum((CASE WHEN (((t1n.c1n)::absint) IS NULL) THEN 1 ELSE 0 END))))) IS NULL)) THEN true ELSE false END ELSE NULL::boolean END)
+         ->  GroupAggregate  (cost=0.00..1324036.49 rows=4 width=20)
+               Group Key: t1.c1, t1.ctid, t1.gp_segment_id
+               ->  Sort  (cost=0.00..1324036.49 rows=4 width=30)
+                     Sort Key: t1.ctid, t1.gp_segment_id
+                     ->  Redistribute Motion 3:3  (slice2; segments: 3)  (cost=0.00..1324036.49 rows=4 width=30)
+                           Hash Key: t1.ctid, t1.gp_segment_id
+                           ->  Result  (cost=0.00..1324036.49 rows=4 width=30)
+                                 ->  HashAggregate  (cost=0.00..1324036.49 rows=4 width=30)
+                                       Group Key: t1.c1, t1.ctid, t1.gp_segment_id
+                                       ->  Nested Loop Left Join  (cost=0.00..1324036.48 rows=17 width=18)
+                                             Join Filter: (((t1.c1)::absint = ((t1n.c1n)::absint)) OR (((t1n.c1n)::absint) IS NULL))
+                                             ->  Seq Scan on t1  (cost=0.00..431.00 rows=4 width=14)
+                                             ->  Result  (cost=0.00..431.00 rows=7 width=8)
+                                                   ->  Materialize  (cost=0.00..431.00 rows=7 width=4)
+                                                         ->  Broadcast Motion 3:3  (slice1; segments: 3)  (cost=0.00..431.00 rows=7 width=4)
+                                                               ->  Result  (cost=0.00..431.00 rows=3 width=4)
+                                                                     ->  Seq Scan on t1n  (cost=0.00..431.00 rows=3 width=4)
+ Planning time: 48.367 ms
+ Optimizer: PQO version 3.17.0
+(22 rows)
+
+select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+ c1 
+----
+(0 rows)
+
 reset search_path;
 drop schema notin cascade;
-NOTICE:  drop cascades to 7 other objects
+NOTICE:  drop cascades to 11 other objects
 DETAIL:  drop cascades to table notin.t1
 drop cascades to table notin.t2
 drop cascades to table notin.t3
@@ -1211,3 +1253,7 @@ drop cascades to table notin.t4
 drop cascades to table notin.t1n
 drop cascades to table notin.g1
 drop cascades to table notin.l1
+drop cascades to type notin.absint
+drop cascades to function notin.iszero(notin.absint)
+drop cascades to function notin.abseq(notin.absint,notin.absint)
+drop cascades to operator notin.=(notin.absint,notin.absint)

src/test/regress/sql/notin.sql

@@ -376,5 +376,18 @@ select c1 from t1 where c1 not in (select c2 from t2 where c2 > 4) and c1 is not
 --
 select c1 from t1 where c1 not in (select c2 from t2 where c2 > 4) and c1 > 2;
 
+-- Test if the equality operator is implemented by a SQL function
+--
+--q45
+--
+create domain absint as int4;
+create function iszero(absint) returns bool as $$ begin return $1::int4 = 0; end; $$ language plpgsql immutable strict;
+create or replace function abseq (absint, absint) returns bool as $$ select iszero(abs($1) - abs($2)); $$ language sql immutable strict;
+create operator = (PROCEDURE = abseq, leftarg=absint, rightarg=absint);
+explain select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+select c1 from t1 where c1::absint not in
+	(select c1n::absint from t1n);
+
 reset search_path;
 drop schema notin cascade;