Convert all remaining geometric operators to new fmgr style. This

allows fixing problems with operators that expected to be able to
return a NULL, such as the '#' line-segment-intersection operator
that tried to return NULL when the two segments don't intersect.
(See, eg, bug report from 1-Nov-99 on pghackers.)  Fix some other
bugs in passing, such as backwards comparison in path_distance().

src/backend/access/rtree/rtproc.c

@@ -3,12 +3,19 @@
  * rtproc.c
  *	  pg_amproc entries for rtrees.
  *
+ * NOTE: for largely-historical reasons, the intersection functions should
+ * return a NULL pointer (*not* an SQL null value) to indicate "no
+ * intersection".  The size functions must be prepared to accept such
+ * a pointer and return 0.  This convention means that only pass-by-reference
+ * data types can be used as the output of the union and intersection
+ * routines, but that's not a big problem.
+ *
+ *
  * Portions Copyright (c) 1996-2000, PostgreSQL, Inc
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/rtree/Attic/rtproc.c,v 1.28 2000/07/29 18:45:52 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/rtree/Attic/rtproc.c,v 1.29 2000/07/30 20:43:40 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -18,29 +25,31 @@
 #include "utils/geo_decls.h"
 
 
-BOX *
-rt_box_union(BOX *a, BOX *b)
+Datum
+rt_box_union(PG_FUNCTION_ARGS)
 {
+	BOX		   *a = PG_GETARG_BOX_P(0);
+	BOX		   *b = PG_GETARG_BOX_P(1);
 	BOX		   *n;
 
-	if ((n = (BOX *) palloc(sizeof(*n))) == (BOX *) NULL)
-		elog(ERROR, "Cannot allocate box for union");
+	n = (BOX *) palloc(sizeof(BOX));
 
 	n->high.x = Max(a->high.x, b->high.x);
 	n->high.y = Max(a->high.y, b->high.y);
 	n->low.x = Min(a->low.x, b->low.x);
 	n->low.y = Min(a->low.y, b->low.y);
 
-	return n;
+	PG_RETURN_BOX_P(n);
 }
 
-BOX *
-rt_box_inter(BOX *a, BOX *b)
+Datum
+rt_box_inter(PG_FUNCTION_ARGS)
 {
+	BOX		   *a = PG_GETARG_BOX_P(0);
+	BOX		   *b = PG_GETARG_BOX_P(1);
 	BOX		   *n;
 
-	if ((n = (BOX *) palloc(sizeof(*n))) == (BOX *) NULL)
-		elog(ERROR, "Cannot allocate box for union");
+	n = (BOX *) palloc(sizeof(BOX));
 
 	n->high.x = Min(a->high.x, b->high.x);
 	n->high.y = Min(a->high.y, b->high.y);
@@ -50,21 +59,26 @@ rt_box_inter(BOX *a, BOX *b)
 	if (n->high.x < n->low.x || n->high.y < n->low.y)
 	{
 		pfree(n);
-		return (BOX *) NULL;
+		/* Indicate "no intersection" by returning NULL pointer */
+		n = NULL;
 	}
 
-	return n;
+	PG_RETURN_BOX_P(n);
 }
 
-void
-rt_box_size(BOX *a, float *size)
+Datum
+rt_box_size(PG_FUNCTION_ARGS)
 {
+	BOX		   *a = PG_GETARG_BOX_P(0);
+	/* NB: size is an output argument */
+	float	   *size = (float *) PG_GETARG_POINTER(1);
+
 	if (a == (BOX *) NULL || a->high.x <= a->low.x || a->high.y <= a->low.y)
 		*size = 0.0;
 	else
 		*size = (float) ((a->high.x - a->low.x) * (a->high.y - a->low.y));
 
-	return;
+	PG_RETURN_VOID();
 }
 
 /*
@@ -75,10 +89,10 @@ rt_box_size(BOX *a, float *size)
  *		as the return type for the size routine, so we no longer need to
  *		have a special return type for big boxes.
  */
-void
-rt_bigbox_size(BOX *a, float *size)
+Datum
+rt_bigbox_size(PG_FUNCTION_ARGS)
 {
-	rt_box_size(a, size);
+	return rt_box_size(fcinfo);
 }
 
 Datum
@@ -105,30 +119,6 @@ rt_poly_union(PG_FUNCTION_ARGS)
 	PG_RETURN_POLYGON_P(p);
 }
 
-Datum
-rt_poly_size(PG_FUNCTION_ARGS)
-{
-	POLYGON	   *a = PG_GETARG_POLYGON_P(0);
-	/* NB: size is an output argument */
-	float	   *size = (float *) PG_GETARG_POINTER(1);
-	double		xdim,
-				ydim;
-
-	if (a == (POLYGON *) NULL ||
-		a->boundbox.high.x <= a->boundbox.low.x ||
-		a->boundbox.high.y <= a->boundbox.low.y)
-		*size = 0.0;
-	else
-	{
-		xdim = (a->boundbox.high.x - a->boundbox.low.x);
-		ydim = (a->boundbox.high.y - a->boundbox.low.y);
-
-		*size = (float) (xdim * ydim);
-	}
-
-	PG_RETURN_VOID();
-}
-
 Datum
 rt_poly_inter(PG_FUNCTION_ARGS)
 {
@@ -146,16 +136,52 @@ rt_poly_inter(PG_FUNCTION_ARGS)
 	p->boundbox.low.x = Max(a->boundbox.low.x, b->boundbox.low.x);
 	p->boundbox.low.y = Max(a->boundbox.low.y, b->boundbox.low.y);
 
-	/* Avoid leaking memory when handed toasted input. */
-	PG_FREE_IF_COPY(a, 0);
-	PG_FREE_IF_COPY(b, 1);
-
 	if (p->boundbox.high.x < p->boundbox.low.x ||
 		p->boundbox.high.y < p->boundbox.low.y)
 	{
 		pfree(p);
-		PG_RETURN_NULL();
+		/* Indicate "no intersection" by returning NULL pointer */
+		p = NULL;
 	}
 
+	/* Avoid leaking memory when handed toasted input. */
+	PG_FREE_IF_COPY(a, 0);
+	PG_FREE_IF_COPY(b, 1);
+
 	PG_RETURN_POLYGON_P(p);
 }
+
+Datum
+rt_poly_size(PG_FUNCTION_ARGS)
+{
+	Pointer		aptr = PG_GETARG_POINTER(0);
+	/* NB: size is an output argument */
+	float	   *size = (float *) PG_GETARG_POINTER(1);
+	POLYGON	   *a;
+	double		xdim,
+				ydim;
+
+	/* Can't just use GETARG because of possibility that input is NULL;
+	 * since POLYGON is toastable, GETARG will try to inspect its value
+	 */
+	if (aptr == NULL)
+	{
+		*size = 0.0;
+		PG_RETURN_VOID();
+	}
+	/* Now safe to apply GETARG */
+	a = PG_GETARG_POLYGON_P(0);
+
+	if (a->boundbox.high.x <= a->boundbox.low.x ||
+		a->boundbox.high.y <= a->boundbox.low.y)
+		*size = 0.0;
+	else
+	{
+		xdim = (a->boundbox.high.x - a->boundbox.low.x);
+		ydim = (a->boundbox.high.y - a->boundbox.low.y);
+
+		*size = (float) (xdim * ydim);
+	}
+
+	PG_RETURN_VOID();
+}

src/backend/access/rtree/rtree.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/rtree/Attic/rtree.c,v 1.52 2000/07/14 22:17:36 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/rtree/Attic/rtree.c,v 1.53 2000/07/30 20:43:40 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -745,13 +745,16 @@ picksplit(Relation r,
 				DatumGetPointer(FunctionCall2(&rtstate->interFn,
 											  PointerGetDatum(datum_alpha),
 											  PointerGetDatum(datum_beta)));
+			/* The interFn may return a NULL pointer (not an SQL null!)
+			 * to indicate no intersection.  sizeFn must cope with this.
+			 */
 			FunctionCall2(&rtstate->sizeFn,
 						  PointerGetDatum(inter_d),
 						  PointerGetDatum(&size_inter));
 			size_waste = size_union - size_inter;
 
-			pfree(union_d);
-
+			if (union_d != (char *) NULL)
+				pfree(union_d);
 			if (inter_d != (char *) NULL)
 				pfree(inter_d);
 
@@ -1051,7 +1054,8 @@ _rtdump(Relation r)
 			itoffno = ItemPointerGetOffsetNumber(&(itup->t_tid));
 			datum = ((char *) itup);
 			datum += sizeof(IndexTupleData);
-			itkey = (char *) box_out((BOX *) datum);
+			itkey = DatumGetCString(DirectFunctionCall1(box_out,
+												PointerGetDatum(datum)));
 			printf("\t[%d] size %d heap <%d,%d> key:%s\n",
 				   offnum, IndexTupleSize(itup), itblkno, itoffno, itkey);
 			pfree(itkey);

src/test/regress/input/create_function_2.source

@@ -45,7 +45,7 @@ CREATE FUNCTION boxarea(box)
 CREATE FUNCTION interpt_pp(path, path)
    RETURNS point
    AS '_OBJWD_/regress_DLSUFFIX_'
-   LANGUAGE 'c';
+   LANGUAGE 'newC';
 
 CREATE FUNCTION reverse_name(name)
    RETURNS name

src/test/regress/output/create_function_2.source

@@ -35,7 +35,7 @@ CREATE FUNCTION boxarea(box)
 CREATE FUNCTION interpt_pp(path, path)
    RETURNS point
    AS '_OBJWD_/regress_DLSUFFIX_'
-   LANGUAGE 'c';
+   LANGUAGE 'newC';
 CREATE FUNCTION reverse_name(name)
    RETURNS name
    AS '_OBJWD_/regress_DLSUFFIX_'

src/test/regress/regress.c

 /*
- * $Header: /cvsroot/pgsql/src/test/regress/regress.c,v 1.42 2000/07/29 18:46:12 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/test/regress/regress.c,v 1.43 2000/07/30 20:43:54 tgl Exp $
  */
 
 #include <float.h>				/* faked on sunos */
@@ -17,10 +17,10 @@
 
 typedef TupleTableSlot *TUPLE;
 
-extern double *regress_dist_ptpath(Point *pt, PATH *path);
-extern double *regress_path_dist(PATH *p1, PATH *p2);
+extern Datum regress_dist_ptpath(PG_FUNCTION_ARGS);
+extern Datum regress_path_dist(PG_FUNCTION_ARGS);
 extern PATH *poly2path(POLYGON *poly);
-extern Point *interpt_pp(PATH *p1, PATH *p2);
+extern Datum interpt_pp(PG_FUNCTION_ARGS);
 extern void regress_lseg_construct(LSEG *lseg, Point *pt1, Point *pt2);
 extern Datum overpaid(PG_FUNCTION_ARGS);
 extern Datum boxarea(PG_FUNCTION_ARGS);
@@ -29,24 +29,22 @@ extern char *reverse_name(char *string);
 /*
 ** Distance from a point to a path
 */
-double *
-regress_dist_ptpath(pt, path)
-Point	   *pt;
-PATH	   *path;
+Datum
+regress_dist_ptpath(PG_FUNCTION_ARGS)
 {
-	double	   *result;
-	double	   *tmp;
+	Point	   *pt = PG_GETARG_POINT_P(0);
+	PATH	   *path = PG_GETARG_PATH_P(1);
+	float8		result = 0.0;	/* keep compiler quiet */
+	float8		tmp;
 	int			i;
 	LSEG		lseg;
 
 	switch (path->npts)
 	{
 		case 0:
-			result = palloc(sizeof(double));
-			*result = Abs((double) DBL_MAX);	/* +infinity */
-			break;
+			PG_RETURN_NULL();
 		case 1:
-			result = point_distance(pt, &path->p[0]);
+			result = point_dt(pt, &path->p[0]);
 			break;
 		default:
 
@@ -55,51 +53,57 @@ PATH	   *path;
 			 * distance from the point to any of its constituent segments.
 			 */
 			Assert(path->npts > 1);
-			result = palloc(sizeof(double));
 			for (i = 0; i < path->npts - 1; ++i)
 			{
 				regress_lseg_construct(&lseg, &path->p[i], &path->p[i + 1]);
-				tmp = dist_ps(pt, &lseg);
-				if (i == 0 || *tmp < *result)
-					*result = *tmp;
-				pfree(tmp);
-
+				tmp = DatumGetFloat8(DirectFunctionCall2(dist_ps,
+													PointPGetDatum(pt),
+													LsegPGetDatum(&lseg)));
+				if (i == 0 || tmp < result)
+					result = tmp;
 			}
 			break;
 	}
-	return result;
+	PG_RETURN_FLOAT8(result);
 }
 
 /* this essentially does a cartesian product of the lsegs in the
    two paths, and finds the min distance between any two lsegs */
-double *
-regress_path_dist(p1, p2)
-PATH	   *p1;
-PATH	   *p2;
+Datum
+regress_path_dist(PG_FUNCTION_ARGS)
 {
-	double	   *min,
-			   *tmp;
+	PATH	   *p1 = PG_GETARG_PATH_P(0);
+	PATH	   *p2 = PG_GETARG_PATH_P(1);
+	bool		have_min = false;
+	float8		min = 0.0;		/* initialize to keep compiler quiet */
+	float8		tmp;
 	int			i,
 				j;
 	LSEG		seg1,
 				seg2;
 
-	regress_lseg_construct(&seg1, &p1->p[0], &p1->p[1]);
-	regress_lseg_construct(&seg2, &p2->p[0], &p2->p[1]);
-	min = lseg_distance(&seg1, &seg2);
-
 	for (i = 0; i < p1->npts - 1; i++)
+	{
 		for (j = 0; j < p2->npts - 1; j++)
 		{
 			regress_lseg_construct(&seg1, &p1->p[i], &p1->p[i + 1]);
 			regress_lseg_construct(&seg2, &p2->p[j], &p2->p[j + 1]);
 
-			if (*min < *(tmp = lseg_distance(&seg1, &seg2)))
-				*min = *tmp;
-			pfree(tmp);
+			tmp = DatumGetFloat8(DirectFunctionCall2(lseg_distance,
+													 LsegPGetDatum(&seg1),
+													 LsegPGetDatum(&seg2)));
+			if (!have_min || tmp < min)
+			{
+				min = tmp;
+				have_min = true;
+			}
 		}
+	}
 
-	return min;
+	if (! have_min)
+		PG_RETURN_NULL();
+
+	PG_RETURN_FLOAT8(min);
 }
 
 PATH *
@@ -124,43 +128,43 @@ POLYGON    *poly;
 											 CStringGetDatum(output)));
 }
 
-/* return the point where two paths intersect.	Assumes that they do. */
-Point *
-interpt_pp(p1, p2)
-PATH	   *p1;
-PATH	   *p2;
+/* return the point where two paths intersect, or NULL if no intersection. */
+Datum
+interpt_pp(PG_FUNCTION_ARGS)
 {
-
-	Point	   *retval;
+	PATH	   *p1 = PG_GETARG_PATH_P(0);
+	PATH	   *p2 = PG_GETARG_PATH_P(1);
 	int			i,
 				j;
 	LSEG		seg1,
 				seg2;
-
-#ifdef NOT_USED
-	LINE	   *ln;
-
-#endif
 	bool		found;			/* We've found the intersection */
 
 	found = false;				/* Haven't found it yet */
 
 	for (i = 0; i < p1->npts - 1 && !found; i++)
+	{
+		regress_lseg_construct(&seg1, &p1->p[i], &p1->p[i + 1]);
 		for (j = 0; j < p2->npts - 1 && !found; j++)
 		{
-			regress_lseg_construct(&seg1, &p1->p[i], &p1->p[i + 1]);
 			regress_lseg_construct(&seg2, &p2->p[j], &p2->p[j + 1]);
-			if (lseg_intersect(&seg1, &seg2))
+			if (DatumGetBool(DirectFunctionCall2(lseg_intersect,
+												 LsegPGetDatum(&seg1),
+												 LsegPGetDatum(&seg2))))
 				found = true;
 		}
+	}
 
-#ifdef NOT_USED
-	ln = line_construct_pp(&seg2.p[0], &seg2.p[1]);
-	retval = interpt_sl(&seg1, ln);
-#endif
-	retval = lseg_interpt(&seg1, &seg2);
+	if (!found)
+		PG_RETURN_NULL();
 
-	return retval;
+	/* Note: DirectFunctionCall2 will kick out an error if lseg_interpt()
+	 * returns NULL, but that should be impossible since we know the two
+	 * segments intersect.
+	 */
+	PG_RETURN_DATUM(DirectFunctionCall2(lseg_interpt,
+										LsegPGetDatum(&seg1),
+										LsegPGetDatum(&seg2)));
 }