Don't use TIDs with high offset numbers in AO tables.

Change the mapping AO segfilenum+rownum to an ItemPointer, so that we
avoid using ItemPointer.ip_posid values higher than 32768. Such offsets
are impossible on heap tables, because you can't fit that many tuples on
a page. In GiST, since PostgreSQL 9.1, we have taken advantage of that by
using 0xfffe (65534) to mark special "invalid" GiST tuples. We can tolerate
that, because those invalid tuples can only appear on internal pages, so
they cannot be confused with AO TIDs, which only appear on leaf pages. But
we will also use of those high values for other similar magic values, in
later version of PostgreSQL, so it seems better to keep clear of them, even
if we could make it work.

To allow binary upgrades of indexes that already contain AO tids with high
offsets, we still allow and handle those, too, in the code to fetch AO
tuples. Also relax the sanity check in GiST code, to not confuse those high
values with invalid tuples.

Fixes https://github.com/greenplum-db/gpdb/issues/6227Reviewed-by: NAshwin Agrawal <aagrawal@pivotal.io>

src/backend/access/aocs/aocsam.c

@@ -699,17 +699,14 @@ ReadNext:
 			}
 		}
 
-		AOTupleIdInit_Init(&aoTupleId);
-		AOTupleIdInit_segmentFileNum(&aoTupleId, curseginfo->segno);
-
 		scan->cur_seg_row++;
 		if (rowNum == INT64CONST(-1))
 		{
-			AOTupleIdInit_rowNum(&aoTupleId, scan->cur_seg_row);
+			AOTupleIdInit(&aoTupleId, curseginfo->segno, scan->cur_seg_row);
 		}
 		else
 		{
-			AOTupleIdInit_rowNum(&aoTupleId, rowNum);
+			AOTupleIdInit(&aoTupleId, curseginfo->segno, rowNum);
 		}
 
 		if (!isSnapshotAny && !AppendOnlyVisimap_IsVisible(&scan->visibilityMap, &aoTupleId))
@@ -972,9 +969,7 @@ aocs_insert_values(AOCSInsertDesc idesc, Datum *d, bool *null, AOTupleId *aoTupl
 
 	Assert(idesc->numSequences >= 0);
 
-	AOTupleIdInit_Init(aoTupleId);
-	AOTupleIdInit_segmentFileNum(aoTupleId, idesc->cur_segno);
-	AOTupleIdInit_rowNum(aoTupleId, idesc->lastSequence);
+	AOTupleIdInit(aoTupleId, idesc->cur_segno, idesc->lastSequence);
 
 	/*
 	 * If the allocated fast sequence numbers are used up, we request for a

src/backend/access/appendonly/appendonly_visimap.c

@@ -338,7 +338,7 @@ AppendOnlyVisimapScan_GetNextInvisible(
 				visiMapScan->isFinished = true;
 				return false;
 			}
-			AOTupleIdInit_Init(tupleId);
+			AOTupleIdSetInvalid(tupleId);
 		}
 
 		if (!AppendOnlyVisimapEntry_GetNextInvisible(

src/backend/access/appendonly/appendonly_visimap_entry.c

@@ -627,13 +627,7 @@ AppendOnlyVisimapEntry_GetNextInvisible(
 	if (offset >= 0)
 	{
 		rowNum = visiMapEntry->firstRowNum + offset;
-		AOTupleIdInit_Init(tupleId);
-		AOTupleIdInit_segmentFileNum(
-									 tupleId,
-									 visiMapEntry->segmentFileNum);
-		AOTupleIdInit_rowNum(
-							 tupleId,
-							 rowNum);
+		AOTupleIdInit(tupleId, visiMapEntry->segmentFileNum, rowNum);
 		return true;
 	}
 	else

src/backend/access/appendonly/appendonly_visimap_udf.c

@@ -89,7 +89,7 @@ gp_aovisimap(PG_FUNCTION_ARGS)
 								   context->aorel->rd_appendonly->visimapidxid,
 								   AccessShareLock,
 								   GetLatestSnapshot());
-		AOTupleIdInit_Init(&context->aoTupleId);
+		AOTupleIdSetInvalid(&context->aoTupleId);
 
 		funcctx->user_fctx = (void *) context;
 

src/backend/access/appendonly/appendonlyam.c

@@ -1011,9 +1011,7 @@ AppendOnlyExecutorReadBlock_ProcessTuple(AppendOnlyExecutorReadBlock *executorRe
 
 	AORelationVersion_CheckValid(formatVersion);
 
-	AOTupleIdInit_Init(aoTupleId);
-	AOTupleIdInit_segmentFileNum(aoTupleId, executorReadBlock->segmentFileNum);
-	AOTupleIdInit_rowNum(aoTupleId, rowNum);
+	AOTupleIdInit(aoTupleId, executorReadBlock->segmentFileNum, rowNum);
 
 	if (slot)
 	{
@@ -3028,9 +3026,7 @@ appendonly_insert(AppendOnlyInsertDesc aoInsertDesc,
 
 	tupleOid = MemTupleGetOid(tup, aoInsertDesc->mt_bind);
 
-	AOTupleIdInit_Init(aoTupleId);
-	AOTupleIdInit_segmentFileNum(aoTupleId, aoInsertDesc->cur_segno);
-	AOTupleIdInit_rowNum(aoTupleId, aoInsertDesc->lastSequence);
+	AOTupleIdInit(aoTupleId, aoInsertDesc->cur_segno, aoInsertDesc->lastSequence);
 
 	/*
 	 * If the allocated fast sequence numbers are used up, we request for a

src/backend/access/appendonly/test/appendonly_visimap_entry_test.c

@@ -17,16 +17,14 @@ test__AppendOnlyVisimapEntry_GetFirstRowNum(void **state)
 	AOTupleId  *tupleId = (AOTupleId *) &fake_ctid;
 
 	/* 5 is less than APPENDONLY_VISIMAP_MAX_RANGE so should get 0 */
-	AOTupleIdInit_Init(tupleId);
-	AOTupleIdInit_segmentFileNum(tupleId, 1);
-	AOTupleIdInit_rowNum(tupleId, 5);
+	AOTupleIdInit(tupleId, 1, 5);
 	expected = 0;
 
 	result = AppendOnlyVisimapEntry_GetFirstRowNum(NULL, tupleId);
 	assert_true(result == expected);
 
 	/* test to make sure we can go above INT32_MAX */
-	AOTupleIdInit_rowNum(tupleId, 3000000000);
+	AOTupleIdInit(tupleId, 1, 3000000000);
 	expected = 2999975936;
 
 	result = AppendOnlyVisimapEntry_GetFirstRowNum(NULL, tupleId);
@@ -40,9 +38,7 @@ test__AppendOnlyVisimapEntry_CoversTuple(void **state)
 	ItemPointerData fake_ctid;
 	AOTupleId  *tupleId = (AOTupleId *) &fake_ctid;
 
-	AOTupleIdInit_Init(tupleId);
-	AOTupleIdInit_segmentFileNum(tupleId, 1);
-	AOTupleIdInit_rowNum(tupleId, 5);
+	AOTupleIdInit(tupleId, 1, 5);
 
 	AppendOnlyVisimapEntry* visiMapEntry = malloc(sizeof(AppendOnlyVisimapEntry));;
 
@@ -68,7 +64,7 @@ test__AppendOnlyVisimapEntry_CoversTuple(void **state)
 	assert_true(result);
 
 	/* Tuple is covered by visimap entry above INT32_MAX row number. */
-	AOTupleIdInit_rowNum(tupleId, 3000000000);
+	AOTupleIdInit(tupleId, 1, 3000000000);
 	visiMapEntry->firstRowNum = 2999975936;
 	result = AppendOnlyVisimapEntry_CoversTuple(visiMapEntry, tupleId);
 	assert_true(result);

src/backend/access/gist/gistget.c

@@ -66,13 +66,15 @@ gistindex_keytest(IndexScanDesc scan,
 	 * If it's a leftover invalid tuple from pre-9.1, treat it as a match with
 	 * minimum possible distances.  This means we'll always follow it to the
 	 * referenced page.
+	 *
+	 * GPDB: the virtual TIDs created for AO tables use the full range of
+	 * offset numbers from 0 to 65535. So a tuple on leaf page that looks like
+	 * an invalid tuple, is actually ok.
 	 */
-	if (GistTupleIsInvalid(tuple))
+	if (!GistPageIsLeaf(page) && GistTupleIsInvalid(tuple))
 	{
 		int			i;
 
-		if (GistPageIsLeaf(page))		/* shouldn't happen */
-			elog(ERROR, "invalid GiST tuple found on leaf page");
 		for (i = 0; i < scan->numberOfOrderBys; i++)
 			so->distances[i] = -get_float8_infinity();
 		return true;

src/backend/executor/nodeBitmapHeapscan.c

@@ -750,12 +750,6 @@ BitmapAppendOnlyNext(BitmapHeapScanState *node)
 		{
 			Assert(node->baos_cindex <= tbmres->ntuples);
 			psuedoHeapOffset = tbmres->offsets[node->baos_cindex];
-
-			/*
-			 * Ensure that the reserved 16-th bit is always ON for offsets from
-			 * lossless bitmap pages [MPP-24326].
-			 */
-			Assert(((uint16)(psuedoHeapOffset & 0x8000)) > 0);
 		}
 
 		/*

src/include/access/appendonlytid.h

@@ -17,31 +17,40 @@
 #include "c.h"
 
 /*
- * AOTupleId is a unique tuple id, specific to AO
- * relation tuples, of the form (segfile#, row#)
+ * AOTupleId is a unique tuple id, specific to AO relation tuples, of the
+ * form (segfile#, row#)
  *
- * *** WARNING *** Some code interprets AOTIDs as HEAPTIDs, and would like
- * AOTIDs has the same ordering as HEAPTIDs.
+ * *** WARNING *** Outside the appendonly AM code, AOTIDs are treated as
+ * HEAPTIDs!  Therefore, AOTupleId struct must have the same size and
+ * alignment as ItemPointer, and we mustn't construct AOTupleIds that would
+ * look like invalid ItemPointers.
  *
- * Begin mischief.  Want to make sure the last 16 bits (uint16) are non-zero so that when
- * the executor asserts the item number is non-zero our AOTID will pass the test.
- * So, we create a reserved bit that is always on.  So, in the case that the segment file
- * number is 0 (utility mode) and the lower part (15 bits) of the row number is 0, our AOTID's
- * last 16 bits will be non-zero because of the always on reserved bit.
+ * The 48 bits available in the struct are divided as follow:
  *
- * Out of the following 48 bits, the 7 leftmost bits stand for which segment file the
- * tuple is in (Limit: 128 (2^8)), the 16th rightmost bit is reserved and always 1,
- * the remaining 40 bits stand for the row within the  segment file
- * (Limit: 1099 trillion (2^40 - 1)).
+ * - the 7 leftmost bits stand for which segment file the tuple is in
+ *   (Limit: 128 (2^8))
  *
+ * - the 16th bit in bytes_4_5 is used to avoid an all-zeros value in the
+ *   field (see below)
  *
- * ***WARNING*** STRUCT PACKING ISSUE.
+ * - the remaining 40 bits stand for the row within the segment file
+ *   (Limit: 1099 trillion (2^40 - 1)).
  *
- *       Previously, we had the uint16 first and the uint32 second.  But on some compilers
- *       the second uint32 would get packed on a 32-bit boundary and the struct would end
- *       up with a hole between the fields...  Arranging the uint16 second is known to work
- *       correctly since the ItemPointerData has the same layout.  And, apparently putting
- *       small fields after big ones is a policy in system catalog struct layouts.
+ * About that 16th bit in 'bytes_4_5': When an AOTupleId is cast to an
+ * ItemPointer, 'bytes_4_5' becomes the 'ip_posid' field in the ItemPointer.
+ * An invalid itempointer is represented by 0 'ip_posid', so we must avoid
+ * that.  To do so, if 'bytes_4_5' would be all zeros, we set the 16th bit.
+ * That occurs when the 15 lower bits of the row number are 0.
+ *
+ * NOTE: Before GPDB6, we *always* set the reserved bit, not only when
+ * 'bytes_4_5' would otherwise be zero.  That was simpler, but caused problems
+ * of its own, because very high offset numbers, which cannot occur in heap
+ * tables, have a special meaning in some places.  See TUPLE_IS_INVALID in
+ * gist_private.h, for example.  In other words, the 'ip_posid' field now uses
+ * values in the range 1..32768, whereas before GPDB 6, it used the range
+ * (32768..65535).  We must still be prepared to deal with the old TIDs, if
+ * the cluster was binary-upgrade from an older version! Because of that, the
+ * accessor macros simply ignore the reserved bit.
  */
 typedef struct AOTupleId
 {
@@ -51,33 +60,31 @@ typedef struct AOTupleId
 
 } AOTupleId;
 
-#define AOTUPLEID_INIT {0,0}
-
 #define AOTupleIdGet_segmentFileNum(h)        ((((h)->bytes_0_1&0xFE00)>>9)) // 7 bits
 #define AOTupleIdGet_makeHeapExecutorHappy(h) (((h)->bytes_4_5&0x8000)) // 1 bit
 #define AOTupleIdGet_rowNum(h) \
 	((((uint64)((h)->bytes_0_1&0x01FF))<<31)|(((uint64)((h)->bytes_2_3))<<15)|(((uint64)((h)->bytes_4_5&0x7FFF))))
  /* top most 25 bits */	/* 15 bits from bytes_4_5 */
 
-/* ~_Init zeroes the 2 regular fields and sets the always on field to 1. */
 static inline void
-AOTupleIdInit_Init(AOTupleId *h)
+AOTupleIdInit(AOTupleId *h, uint16 segfilenum, uint64 rownum)
 {
-	h->bytes_0_1 = 0;
-	h->bytes_2_3 = 0;
-	h->bytes_4_5 = 0x8000;
-}
-static inline void
-AOTupleIdInit_segmentFileNum(AOTupleId *h, uint16 e)
-{
-	h->bytes_0_1 |= ((uint16) (0x007F & e)) << 9;
+	h->bytes_0_1 = ((uint16) (0x007F & segfilenum)) << 9;
+	h->bytes_0_1 |= (uint16) ((INT64CONST(0x000000FFFFFFFFFF) & rownum) >> 31);
+	h->bytes_2_3 = (uint16) ((INT64CONST(0x000000007FFFFFFF) & rownum) >> 15);
+
+	/* If 'bytes_4_5' would otherwise be all-zero, set the reserved bit. */
+	if ((0x7FFF & rownum) == 0)
+		h->bytes_4_5 = 0x8000;
+	else
+		h->bytes_4_5 = 0x7FFF & rownum;
 }
+
+/* like ItemPointerSetInvalid */
 static inline void
-AOTupleIdInit_rowNum(AOTupleId *h, uint64 e)
+AOTupleIdSetInvalid(AOTupleId *h)
 {
-	h->bytes_0_1 |= (uint16) ((INT64CONST(0x000000FFFFFFFFFF) & e) >> 31);
-	h->bytes_2_3 |= (uint16) ((INT64CONST(0x000000007FFFFFFF) & e) >> 15);
-	h->bytes_4_5 |= 0x7FFF & e;
+	ItemPointerSetInvalid((ItemPointer) h);
 }
 
 #define AOTupleId_MaxRowNum            INT64CONST(1099511627775) 		// 40 bits, or 1099511627775 (1099 trillion).

src/test/isolation2/output/uao/parallel_delete.source

@@ -46,8 +46,8 @@ BEGIN
 3: COMMIT;
 COMMIT
 2U: SELECT * FROM gp_toolkit.__gp_aovisimap('ao');
- tid              | segno | row_num 
-------------------+-------+---------
- (33554432,32769) | 1     | 1       
- (33554432,32770) | 1     | 2       
+ tid          | segno | row_num 
+--------------+-------+---------
+ (33554432,1) | 1     | 1       
+ (33554432,2) | 1     | 2       
 (2 rows)

src/test/isolation2/output/uao/parallel_delete_optimizer.source

@@ -48,8 +48,8 @@ BEGIN
 3: COMMIT;
 COMMIT
 2U: SELECT * FROM gp_toolkit.__gp_aovisimap('ao');
- tid              | segno | row_num 
-------------------+-------+---------
- (33554432,32769) | 1     | 1       
- (33554432,32770) | 1     | 2       
+ tid          | segno | row_num 
+--------------+-------+---------
+ (33554432,1) | 1     | 1       
+ (33554432,2) | 1     | 2       
 (2 rows)

src/test/isolation2/output/uao/parallel_update.source

@@ -45,8 +45,8 @@ COMMIT
  4 | 4  
 (4 rows)
 2U: SELECT * FROM gp_toolkit.__gp_aovisimap('ao');
- tid              | segno | row_num 
-------------------+-------+---------
- (33554432,32769) | 1     | 1       
- (33554432,32770) | 1     | 2       
+ tid          | segno | row_num 
+--------------+-------+---------
+ (33554432,1) | 1     | 1       
+ (33554432,2) | 1     | 2       
 (2 rows)

src/test/isolation2/output/uao/parallel_update_optimizer.source

@@ -47,8 +47,8 @@ COMMIT
  4 | 4  
 (4 rows)
 2U: SELECT * FROM gp_toolkit.__gp_aovisimap('ao');
- tid              | segno | row_num 
-------------------+-------+---------
- (33554432,32769) | 1     | 1       
- (33554432,32770) | 1     | 2       
+ tid          | segno | row_num 
+--------------+-------+---------
+ (33554432,1) | 1     | 1       
+ (33554432,2) | 1     | 2       
 (2 rows)

src/test/regress/expected/gp_toolkit_ao_funcs.out

@@ -70,10 +70,10 @@ SELECT count(*) FROM gp_toolkit.__gp_aoseg('toolkit_ao_test');
 SELECT (t).* FROM (
   SELECT gp_toolkit.__gp_aovisimap('toolkit_ao_test') AS t FROM gp_dist_random('gp_id')
 ) AS x;
-       tid        | segno | row_num 
-------------------+-------+---------
- (33554432,32769) |     1 |       1
- (33554432,32770) |     1 |       2
+     tid      | segno | row_num 
+--------------+-------+---------
+ (33554432,1) |     1 |       1
+ (33554432,2) |     1 |       2
 (2 rows)
 
 SELECT (t).segno, (t).first_row_num, (t).hidden_tupcount >= 1 as hidden_tupcount_nonzero, (t).bitmap like '01%' as bitmap_starts_with_01 FROM (

src/test/regress/expected/qp_gist_indexes3.out

@@ -286,6 +286,25 @@ CREATE INDEX GistIndex14a ON GistTable14 USING HASH (id);
 ERROR:  hash indexes are not supported
 CREATE INDEX GistIndex14b ON GistTable14 USING HASH (property);
 ERROR:  hash indexes are not supported
+-- ----------------------------------------------------------------------
+-- Test GiST index on an AO table
+-- ----------------------------------------------------------------------
+------------------------------------------------------------------------------
+-- PURPOSE:
+--     Test the logic in forming TIDs for AO tuples. Before GPDB 6, the
+--     offset numbers in AO tids ran from 32768 to 65535, which collided
+--     with the special offsets used in GiST to mark invalid tuples. This
+--     test reproduced an error with that.
+------------------------------------------------------------------------------
+create table ao_points (i int, p point) with (appendonly=true) distributed by(i);
+create index on ao_points using gist (p);
+insert into ao_points select 1, point(g,g) from generate_series(1, 70000) g;
+select count(*) from ao_points where p <@ box('(32600,32600)', '(32800,32800)');
+ count 
+-------
+   201
+(1 row)
+
 -- ----------------------------------------------------------------------
 -- Test: teardown.sql
 -- ----------------------------------------------------------------------

src/test/regress/expected/qp_gist_indexes3_optimizer.out

@@ -290,6 +290,25 @@ CREATE INDEX GistIndex14a ON GistTable14 USING HASH (id);
 ERROR:  hash indexes are not supported
 CREATE INDEX GistIndex14b ON GistTable14 USING HASH (property);
 ERROR:  hash indexes are not supported
+-- ----------------------------------------------------------------------
+-- Test GiST index on an AO table
+-- ----------------------------------------------------------------------
+------------------------------------------------------------------------------
+-- PURPOSE:
+--     Test the logic in forming TIDs for AO tuples. Before GPDB 6, the
+--     offset numbers in AO tids ran from 32768 to 65535, which collided
+--     with the special offsets used in GiST to mark invalid tuples. This
+--     test reproduced an error with that.
+------------------------------------------------------------------------------
+create table ao_points (i int, p point) with (appendonly=true) distributed by(i);
+create index on ao_points using gist (p);
+insert into ao_points select 1, point(g,g) from generate_series(1, 70000) g;
+select count(*) from ao_points where p <@ box('(32600,32600)', '(32800,32800)');
+ count 
+-------
+   201
+(1 row)
+
 -- ----------------------------------------------------------------------
 -- Test: teardown.sql
 -- ----------------------------------------------------------------------

src/test/regress/expected/qp_misc_jiras.out

@@ -3764,17 +3764,17 @@ create table qp_misc_jiras.test_ao (i int, j int) with (appendonly=true);
 NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table.
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
 insert into qp_misc_jiras.test_ao values (0, 0);
-explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
                                                              QUERY PLAN                                                             
 ------------------------------------------------------------------------------------------------------------------------------------
  Gather Motion 3:1  (slice1; segments: 3)  (cost=0.00..13.91 rows=1 width=8)
    ->  Seq Scan on test_ao  (cost=0.00..13.91 rows=1 width=8)
-         Filter: gp_segment_id >= 0 AND ctid = '(33554432,32769)'::tid
+         Filter: gp_segment_id >= 0 AND ctid = '(33554432,1)'::tid
  Settings:  enable_bitmapscan=off; enable_groupagg=on; enable_indexscan=on; enable_seqscan=off; optimizer=off; optimizer_segments=3
  Optimizer status: legacy query optimizer
 (5 rows)
 
-select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
  i | j 
 ---+---
  0 | 0
@@ -3787,17 +3787,17 @@ create table qp_misc_jiras.test_co (i int, j int) with (appendonly=true, orienta
 NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table.
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
 insert into qp_misc_jiras.test_co values (0, 0);
-explain select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
                                                              QUERY PLAN                                                             
 ------------------------------------------------------------------------------------------------------------------------------------
  Gather Motion 3:1  (slice1; segments: 3)  (cost=0.00..13.91 rows=1 width=8)
    ->  Seq Scan on test_co  (cost=0.00..13.91 rows=1 width=8)
-         Filter: gp_segment_id >= 0 AND ctid = '(33554432,32769)'::tid
+         Filter: gp_segment_id >= 0 AND ctid = '(33554432,1)'::tid
  Settings:  enable_bitmapscan=off; enable_groupagg=on; enable_indexscan=on; enable_seqscan=off; optimizer=off; optimizer_segments=3
  Optimizer status: legacy query optimizer
 (5 rows)
 
-select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
  i | j 
 ---+---
  0 | 0

src/test/regress/expected/qp_misc_jiras_optimizer.out

@@ -3796,17 +3796,17 @@ create table qp_misc_jiras.test_ao (i int, j int) with (appendonly=true);
 NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table.
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
 insert into qp_misc_jiras.test_ao values (0, 0);
-explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
                                                             QUERY PLAN                                                             
 -----------------------------------------------------------------------------------------------------------------------------------
  Gather Motion 3:1  (slice1; segments: 3)  (cost=0.00..431.00 rows=1 width=8)
    ->  Seq Scan on test_ao  (cost=0.00..431.00 rows=1 width=8)
-         Filter: ctid = '(33554432,32769)'::tid AND gp_segment_id >= 0
+         Filter: ctid = '(33554432,1)'::tid AND gp_segment_id >= 0
  Settings:  enable_bitmapscan=off; enable_groupagg=on; enable_indexscan=on; enable_seqscan=off; optimizer=on; optimizer_segments=3
  Optimizer status: PQO version 1.624
 (5 rows)
 
-select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
  i | j 
 ---+---
  0 | 0
@@ -3819,17 +3819,17 @@ create table qp_misc_jiras.test_co (i int, j int) with (appendonly=true, orienta
 NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'i' as the Greenplum Database data distribution key for this table.
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
 insert into qp_misc_jiras.test_co values (0, 0);
-explain select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
                                                             QUERY PLAN                                                             
 -----------------------------------------------------------------------------------------------------------------------------------
  Gather Motion 3:1  (slice1; segments: 3)  (cost=0.00..431.00 rows=1 width=8)
    ->  Seq Scan on test_co  (cost=0.00..431.00 rows=1 width=8)
-         Filter: ctid = '(33554432,32769)'::tid AND gp_segment_id >= 0
+         Filter: ctid = '(33554432,1)'::tid AND gp_segment_id >= 0
  Settings:  enable_bitmapscan=off; enable_groupagg=on; enable_indexscan=on; enable_seqscan=off; optimizer=on; optimizer_segments=3
  Optimizer status: PQO version 1.624
 (5 rows)
 
-select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
  i | j 
 ---+---
  0 | 0

src/test/regress/output/aocs.source

@@ -935,9 +935,9 @@ select count(*) from gp_toolkit.__gp_aocsseg('tenk_aocs1') where modcount > 1;
 CREATE TABLE syscoltest(a int) WITH (appendonly=true, orientation=column);
 INSERT INTO syscoltest VALUES(1);
 SELECT ctid FROM syscoltest;
-       ctid       
-------------------
- (33554432,32769)
+     ctid     
+--------------
+ (33554432,1)
 (1 row)
 
 DROP TABLE syscoltest;

src/test/regress/output/appendonly.source

@@ -1058,9 +1058,9 @@ NOTICE:  Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as
 HINT:  The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew.
 INSERT INTO syscoltest VALUES(1);
 SELECT ctid FROM syscoltest;
-       ctid       
-------------------
- (33554432,32769)
+     ctid     
+--------------
+ (33554432,1)
 (1 row)
 
 DROP TABLE syscoltest;

src/test/regress/sql/qp_gist_indexes3.sql

@@ -258,6 +258,24 @@ CREATE TABLE GistTable14 (
 CREATE INDEX GistIndex14a ON GistTable14 USING HASH (id);
 CREATE INDEX GistIndex14b ON GistTable14 USING HASH (property);
 
+-- ----------------------------------------------------------------------
+-- Test GiST index on an AO table
+-- ----------------------------------------------------------------------
+
+------------------------------------------------------------------------------
+-- PURPOSE:
+--     Test the logic in forming TIDs for AO tuples. Before GPDB 6, the
+--     offset numbers in AO tids ran from 32768 to 65535, which collided
+--     with the special offsets used in GiST to mark invalid tuples. This
+--     test reproduced an error with that.
+------------------------------------------------------------------------------
+create table ao_points (i int, p point) with (appendonly=true) distributed by(i);
+create index on ao_points using gist (p);
+
+insert into ao_points select 1, point(g,g) from generate_series(1, 70000) g;
+
+select count(*) from ao_points where p <@ box('(32600,32600)', '(32800,32800)');
+
 
 -- ----------------------------------------------------------------------
 -- Test: teardown.sql

src/test/regress/sql/qp_misc_jiras.sql

@@ -2333,16 +2333,16 @@ select  * from qp_misc_jiras.test_heap where ctid='(0,1)' and gp_segment_id >= 0
 --
 create table qp_misc_jiras.test_ao (i int, j int) with (appendonly=true);
 insert into qp_misc_jiras.test_ao values (0, 0);
-explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
-select  * from qp_misc_jiras.test_ao where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_ao where ctid='(33554432,1)' and gp_segment_id >= 0;
 
 --
 -- CO table. TidScan should not be used.
 --
 create table qp_misc_jiras.test_co (i int, j int) with (appendonly=true, orientation=column);
 insert into qp_misc_jiras.test_co values (0, 0);
-explain select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
-select  * from qp_misc_jiras.test_co where ctid='(33554432,32769)' and gp_segment_id >= 0;
+explain select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
+select  * from qp_misc_jiras.test_co where ctid='(33554432,1)' and gp_segment_id >= 0;
 
 -- This is to verify MPP-10856: test gp_enable_explain_allstat
 set gp_enable_explain_allstat=on;