Implement NUMERIC upgrade for AOCS versions < 8.3

8.2->8.3 upgrade of NUMERIC types was implemented for row-oriented AO
tables, but not column-oriented. Correct that here.

Store upgraded Datum data in a per-DatumStream buffer, to avoid
"upgrading" the same data multiple times (multiple tuples may be
pointing at the same data buffer, for example with RLE compression).
Cache the column's base type in the DatumStreamRead struct.
Co-authored-by: NTaylor Vesely <tvesely@pivotal.io>

src/backend/access/aocs/aocsam.c

@@ -542,6 +542,80 @@ aocs_endscan(AOCSScanDesc scan)
 	pfree(scan);
 }
 
+/*
+ * Upgrades a Datum value from a previous version of the AOCS page format. The
+ * DatumStreamRead that is passed must correspond to the column being upgraded.
+ */
+static void upgrade_datum_impl(DatumStreamRead *ds, int attno, Datum values[],
+							   bool isnull[], int formatversion)
+{
+	bool 	convert_numeric = false;
+
+	if (PG82NumericConversionNeeded(formatversion))
+	{
+		/*
+		 * On the first call for this DatumStream, figure out if this column is
+		 * a numeric, or a domain over numerics.
+		 *
+		 * TODO: consolidate this code with upgrade_tuple() in appendonlyam.c.
+		 */
+		if (!OidIsValid(ds->baseTypeOid))
+		{
+			ds->baseTypeOid = getBaseType(ds->typeInfo.typid);
+		}
+
+		/* If this Datum is a numeric, we need to convert it. */
+		convert_numeric = (ds->baseTypeOid == NUMERICOID) && !isnull[attno];
+	}
+
+	if (convert_numeric)
+	{
+		/*
+		 * Before PostgreSQL 8.3, the n_weight and n_sign_dscale fields were the
+		 * other way 'round. Swap them.
+		 */
+		Datum 		datum;
+		char	   *numericdata;
+		char	   *upgradedata;
+		size_t		datalen;
+		uint16		tmp;
+
+		/*
+		 * We need to make a copy of this data so that any other tuples pointing
+		 * to it won't be affected. Store it in the upgrade space for this
+		 * DatumStream.
+		 */
+		datum = values[attno];
+		datalen = VARSIZE_ANY(DatumGetPointer(datum));
+
+		upgradedata = datumstreamread_get_upgrade_space(ds, datalen);
+		memcpy(upgradedata, DatumGetPointer(datum), datalen);
+
+		/* Swap the fields. */
+		numericdata = VARDATA_ANY(upgradedata);
+
+		memcpy(&tmp, &numericdata[0], 2);
+		memcpy(&numericdata[0], &numericdata[2], 2);
+		memcpy(&numericdata[2], &tmp, 2);
+
+		/* Re-point the Datum to the upgraded numeric. */
+		values[attno] = PointerGetDatum(upgradedata);
+	}
+}
+
+static void upgrade_datum_scan(AOCSScanDesc scan, int attno, Datum values[],
+							   bool isnull[], int formatversion)
+{
+	upgrade_datum_impl(scan->ds[attno], attno, values, isnull, formatversion);
+}
+
+static void upgrade_datum_fetch(AOCSFetchDesc fetch, int attno, Datum values[],
+								bool isnull[], int formatversion)
+{
+	upgrade_datum_impl(fetch->datumStreamFetchDesc[attno]->datumStream, attno,
+					   values, isnull, formatversion);
+}
+
 void
 aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 {
@@ -561,6 +635,8 @@ aocs_getnext(AOCSScanDesc scan, ScanDirection direction, TupleTableSlot *slot)
 
 	while (1)
 	{
+		AOCSFileSegInfo *curseginfo;
+
 ReadNext:
 		/* If necessary, open next seg */
 		if (scan->cur_seg < 0 || err < 0)
@@ -577,6 +653,7 @@ ReadNext:
 		}
 
 		Assert(scan->cur_seg >= 0);
+		curseginfo = scan->seginfo[scan->cur_seg];
 
 		/* Read from cur_seg */
 		for (i = 0; i < scan->num_proj_atts; i++)
@@ -607,6 +684,15 @@ ReadNext:
 			 */
 			datumstreamread_get(scan->ds[attno], &d[attno], &null[attno]);
 
+			/*
+			 * Perform any required upgrades on the Datum we just fetched.
+			 */
+			if (curseginfo->formatversion < AORelationVersion_GetLatest())
+			{
+				upgrade_datum_scan(scan, attno, d, null,
+								   curseginfo->formatversion);
+			}
+
 			if (rowNum == INT64CONST(-1) &&
 				scan->ds[attno]->blockFirstRowNum != INT64CONST(-1))
 			{
@@ -617,8 +703,7 @@ ReadNext:
 		}
 
 		AOTupleIdInit_Init(&aoTupleId);
-		AOTupleIdInit_segmentFileNum(&aoTupleId,
-									 scan->seginfo[scan->cur_seg]->segno);
+		AOTupleIdInit_segmentFileNum(&aoTupleId, curseginfo->segno);
 
 		scan->cur_seg_row++;
 		if (rowNum == INT64CONST(-1))
@@ -1004,8 +1089,18 @@ fetchFromCurrentBlock(AOCSFetchDesc aocsFetchDesc,
 	{
 		Datum	   *values = slot_get_values(slot);
 		bool	   *nulls = slot_get_isnull(slot);
+		int			formatversion = datumStream->ao_read.formatVersion;
 
 		datumstreamread_get(datumStream, &(values[colno]), &(nulls[colno]));
+
+		/*
+		 * Perform any required upgrades on the Datum we just fetched.
+		 */
+		if (formatversion < AORelationVersion_GetLatest())
+		{
+			upgrade_datum_fetch(aocsFetchDesc, colno, values, nulls,
+								formatversion);
+		}
 	}
 	else
 	{

src/backend/access/appendonly/appendonlyam.c

@@ -911,7 +911,7 @@ upgrade_tuple(AppendOnlyExecutorReadBlock *executorReadBlock,
 			executorReadBlock->numNumericAtts = n;
 		}
 
-		/* If there were any numeric columns, we need to conver them. */
+		/* If there were any numeric columns, we need to convert them. */
 		if (executorReadBlock->numNumericAtts > 0)
 			convert_numerics = true;
 	}

src/backend/utils/datumstream/datumstream.c

@@ -770,6 +770,8 @@ destroy_datumstreamread(DatumStreamRead * ds)
 
 	if (ds->large_object_buffer)
 		pfree(ds->large_object_buffer);
+	if (ds->datum_upgrade_buffer)
+		pfree(ds->datum_upgrade_buffer);
 
 	AppendOnlyStorageRead_FinishSession(&ds->ao_read);
 
@@ -1583,3 +1585,33 @@ datumstreamread_find_block(DatumStreamRead * datumStream,
 
 	return true;
 }
+
+/*
+ * Ensures that the stream's datum_upgrade_buffer is at least len bytes long.
+ * Returns a pointer to the (possibly newly allocated) upgrade buffer space. If
+ * additional space is needed, it will be allocated in the stream's memory
+ * context.
+ */
+void *
+datumstreamread_get_upgrade_space(DatumStreamRead *ds, size_t len)
+{
+	if (ds->datum_upgrade_buffer_size < len)
+	{
+		MemoryContext oldcontext = MemoryContextSwitchTo(ds->memctxt);
+
+		/*
+		 * FIXME: looks like at least one realloc() implementation can't handle
+		 * NULL pointers?
+		 */
+		if (ds->datum_upgrade_buffer)
+			ds->datum_upgrade_buffer = repalloc(ds->datum_upgrade_buffer, len);
+		else
+			ds->datum_upgrade_buffer = palloc(len);
+
+		ds->datum_upgrade_buffer_size = len;
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	return ds->datum_upgrade_buffer;
+}

src/include/utils/datumstream.h

@@ -128,6 +128,14 @@ typedef struct DatumStreamRead
 	uint8	   *large_object_buffer;
 	int32		large_object_buffer_size;
 
+	/*
+	 * Temporary space for storing a Datum that has to be upgraded from a prior
+	 * AO format version but can't be upgraded in place. Allocated only as
+	 * needed, and holds only one value at a time.
+	 */
+	void	   *datum_upgrade_buffer;
+	size_t		datum_upgrade_buffer_size;
+
 	/* EOF of current file */
 	int64		eof;
 	int64		eofUncompress;
@@ -141,6 +149,12 @@ typedef struct DatumStreamRead
 
 	DatumStreamTypeInfo typeInfo;
 
+	/*
+	 * Cached base type, to assist with numeric upgrades. Initialized lazily
+	 * when needed.
+	 */
+	Oid			baseTypeOid;
+
 	bool		rle_can_have_compression;
 	bool		delta_can_have_compression;
 
@@ -300,6 +314,8 @@ extern void datumstreamread_rewind_block(DatumStreamRead * datumStream);
 extern bool datumstreamread_find_block(DatumStreamRead * datumStream,
 						   DatumStreamFetchDesc datumStreamFetchDesc,
 						   int64 rowNum);
+extern void *datumstreamread_get_upgrade_space(DatumStreamRead *datumStream,
+											   size_t len);
 
 /*
  * MPP-17061: make sure datumstream_read_block_info was called first for the CO block

src/test/isolation2/.gitignore

@@ -2,6 +2,9 @@ regression.diffs
 regression.out
 */dummy.sql
 */dummy.out
+sql/ao_upgrade.sql
+expected/ao_upgrade.out
+expected/ao_upgrade_optimizer.out
 
 # Local binaries and symbolic links
 /pg_isolation2_regress

src/test/isolation2/input/ao_upgrade.source

+-- convert_to_v4 takes a modern numeric and converts it into a GPDB4 numeric. 
+CREATE OR REPLACE FUNCTION convert_to_v4(value numeric)
+	RETURNS numeric
+	AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'convertNumericToGPDB4'
+	LANGUAGE C
+	RETURNS NULL ON NULL INPUT;
+
+-- set_ao_formatversion forces an AO[CO] format to a specific version (the last
+-- argument is set to true for a column-oriented table, and false otherwise).
+CREATE OR REPLACE FUNCTION set_ao_formatversion(aosegrel oid, version smallint, isaocs bool)
+	RETURNS bool
+	AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'setAOFormatVersion'
+	LANGUAGE C
+	RETURNS NULL ON NULL INPUT;
+
+DROP TABLE IF EXISTS ao_upgrade_test;
+DROP TABLE IF EXISTS aocs_upgrade_test;
+
+CREATE TABLE ao_upgrade_test (rowid int, n numeric) WITH (appendonly=true);
+CREATE TABLE aocs_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column);
+CREATE TABLE aocs_rle_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column, compresstype=RLE_TYPE);
+
+INSERT INTO ao_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+INSERT INTO aocs_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+
+-- For the RLE test case, insert a bunch of identical numerics so they will be
+-- run-length compressed.
+INSERT INTO aocs_rle_upgrade_test (SELECT a, convert_to_v4(9 !) FROM generate_series(1, 10) a);
+
+-- Check to make sure convert_to_v4() does what we expect; without setting the
+-- AO format version back to GPDB4, the numerics we inserted should be mangled.
+SELECT * FROM ao_upgrade_test;
+SELECT * FROM aocs_upgrade_test;
+SELECT * FROM aocs_rle_upgrade_test;
+
+-- Downgrade to GPDB4 (AO version 2).
+--start_ignore
+*U: SELECT set_ao_formatversion(
+		(SELECT segrelid FROM pg_appendonly WHERE relid = 'ao_upgrade_test'::regclass),
+		2::smallint, false);
+*U: SELECT set_ao_formatversion(
+		(SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_upgrade_test'::regclass),
+		2::smallint, true);
+*U: SELECT set_ao_formatversion(
+		(SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_rle_upgrade_test'::regclass),
+		2::smallint, true);
+--end_ignore
+
+-- Scan test. The numerics should be fixed again.
+SELECT * FROM ao_upgrade_test;
+SELECT * FROM aocs_upgrade_test;
+SELECT * FROM aocs_rle_upgrade_test;
+
+-- Fetch test. To force fetches, we'll add bitmap indexes and disable sequential
+-- scan.
+CREATE INDEX ao_bitmap_index ON ao_upgrade_test USING bitmap(n);
+CREATE INDEX aocs_bitmap_index ON aocs_upgrade_test USING bitmap(n);
+CREATE INDEX aocs_rle_bitmap_index ON aocs_rle_upgrade_test USING bitmap(n);
+
+SET enable_seqscan TO off;
+
+-- Ensure we're using a bitmap scan for our tests. Upgrade note to developers:
+-- the only thing that this test needs to verify is that a fetch-based scan is
+-- in use. Other diffs are fine.
+EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+
+SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+
+RESET enable_seqscan;
+
+DROP FUNCTION convert_to_v4(numeric);
+DROP FUNCTION set_ao_formatversion(oid, smallint, bool);

src/test/isolation2/isolation2_regress.c

@@ -3,14 +3,19 @@
 #include "tablefuncapi.h"
 #include "miscadmin.h"
 
+#include "access/aocssegfiles.h"
 #include "access/heapam.h"
 #include "storage/bufmgr.h"
 
 PG_MODULE_MAGIC;
 
 extern void flush_relation_buffers(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1(flush_relation_buffers);
 
+/* numeric upgrade tests */
+extern Datum convertNumericToGPDB4(PG_FUNCTION_ARGS);
+extern Datum setAOFormatVersion(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(flush_relation_buffers);
 void
 flush_relation_buffers(PG_FUNCTION_ARGS)
 {
@@ -20,3 +25,95 @@ flush_relation_buffers(PG_FUNCTION_ARGS)
 	heap_close(r, AccessShareLock);
 }
 
+/* Mangle a numeric Datum to match the GPDB4 (Postgres 8.2) format. */
+PG_FUNCTION_INFO_V1(convertNumericToGPDB4);
+Datum
+convertNumericToGPDB4(PG_FUNCTION_ARGS)
+{
+	Datum	numeric = PG_GETARG_DATUM(0);
+	void   *varlena = DatumGetPointer(numeric);
+	size_t	datalen;
+	void   *newvarlena;
+	char  *newdata;
+	uint16	tmp;
+
+	datalen = VARSIZE_ANY(varlena);
+	newvarlena = palloc(datalen);
+	memcpy(newvarlena, varlena, datalen);
+
+	newdata = VARDATA_ANY(newvarlena);
+
+	memcpy(&tmp, &newdata[0], 2);
+	memcpy(&newdata[0], &newdata[2], 2);
+	memcpy(&newdata[2], &tmp, 2);
+
+	PG_RETURN_POINTER(newvarlena);
+}
+
+/* Override the format version for an AO/CO table. */
+PG_FUNCTION_INFO_V1(setAOFormatVersion);
+Datum
+setAOFormatVersion(PG_FUNCTION_ARGS)
+{
+	Oid				aosegrelid = PG_GETARG_OID(0);
+	int16			formatversion = PG_GETARG_INT16(1);
+	bool			columnoriented = PG_GETARG_BOOL(2);
+	Relation		aosegrel;
+	HeapScanDesc	scan;
+	HeapTuple		oldtuple;
+	HeapTuple		newtuple;
+	TupleDesc		tupdesc;
+	Datum		   *values;
+	bool		   *isnull;
+	bool		   *replace;
+	int				natts;
+	int				formatversion_attnum;
+
+	/*
+	 * The segment descriptor's rowtype is different for row- and
+	 * column-oriented tables.
+	 */
+	natts = columnoriented ? Natts_pg_aocsseg : Natts_pg_aoseg;
+	formatversion_attnum = columnoriented ? Anum_pg_aocs_formatversion :
+											Anum_pg_aoseg_formatversion;
+
+	/* Create our replacement attribute. */
+	values = palloc(sizeof(Datum) * natts);
+	isnull = palloc0(sizeof(bool) * natts);
+	replace = palloc0(sizeof(bool) * natts);
+
+	values[formatversion_attnum - 1] = Int16GetDatum(formatversion);
+	replace[formatversion_attnum - 1] = true;
+
+	/* Open the segment descriptor table. */
+	aosegrel = heap_open(aosegrelid, RowExclusiveLock);
+
+	if (!RelationIsValid(aosegrel))
+		elog(ERROR, "could not open aoseg table with OID %d", (int) aosegrelid);
+
+	tupdesc = RelationGetDescr(aosegrel);
+
+	/* Try to sanity-check a little bit... */
+	if (tupdesc->natts != natts)
+		elog(ERROR, "table with OID %d does not appear to be an aoseg table",
+			 (int) aosegrelid);
+
+	/* Scan over the rows, overriding the formatversion for each entry. */
+	scan = heap_beginscan(aosegrel, SnapshotNow, 0, NULL);
+	while ((oldtuple = heap_getnext(scan, ForwardScanDirection)) != NULL)
+	{
+		newtuple = heap_modify_tuple(oldtuple, tupdesc, values, isnull, replace);
+		simple_heap_update(aosegrel, &oldtuple->t_self, newtuple);
+		pfree(newtuple);
+	}
+	heap_endscan(scan);
+
+	/* Done. Clean up. */
+	heap_close(aosegrel, RowExclusiveLock);
+
+	pfree(replace);
+	pfree(isnull);
+	pfree(values);
+
+	PG_RETURN_BOOL(true);
+}

src/test/isolation2/output/ao_upgrade.source

+-- convert_to_v4 takes a modern numeric and converts it into a GPDB4 numeric.
+CREATE OR REPLACE FUNCTION convert_to_v4(value numeric) RETURNS numeric AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'convertNumericToGPDB4' LANGUAGE C RETURNS NULL ON NULL INPUT;
+CREATE
+
+-- set_ao_formatversion forces an AO[CO] format to a specific version (the last
+-- argument is set to true for a column-oriented table, and false otherwise).
+CREATE OR REPLACE FUNCTION set_ao_formatversion(aosegrel oid, version smallint, isaocs bool) RETURNS bool AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'setAOFormatVersion' LANGUAGE C RETURNS NULL ON NULL INPUT;
+CREATE
+
+DROP TABLE IF EXISTS ao_upgrade_test;
+DROP
+DROP TABLE IF EXISTS aocs_upgrade_test;
+DROP
+
+CREATE TABLE ao_upgrade_test (rowid int, n numeric) WITH (appendonly=true);
+CREATE
+CREATE TABLE aocs_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column);
+CREATE
+CREATE TABLE aocs_rle_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column, compresstype=RLE_TYPE);
+CREATE
+
+INSERT INTO ao_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+INSERT 10
+INSERT INTO aocs_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+INSERT 10
+
+-- For the RLE test case, insert a bunch of identical numerics so they will be
+-- run-length compressed.
+INSERT INTO aocs_rle_upgrade_test (SELECT a, convert_to_v4(9 !) FROM generate_series(1, 10) a);
+INSERT 10
+
+-- Check to make sure convert_to_v4() does what we expect; without setting the
+-- AO format version back to GPDB4, the numerics we inserted should be mangled.
+SELECT * FROM ao_upgrade_test;
+rowid|n     
+-----+------
+3    |4.0   
+4    |36.3  
+5    |362.9 
+6    |3991.7
+7    |4.79  
+8    |62.27 
+9    |871.78
+10   |1.308 
+1    |720   
+2    |5040  
+(10 rows)
+SELECT * FROM aocs_upgrade_test;
+rowid|n     
+-----+------
+3    |4.0   
+4    |36.3  
+5    |362.9 
+6    |3991.7
+7    |4.79  
+8    |62.27 
+9    |871.78
+10   |1.308 
+1    |720   
+2    |5040  
+(10 rows)
+SELECT * FROM aocs_rle_upgrade_test;
+rowid|n   
+-----+----
+8    |36.3
+9    |36.3
+10   |36.3
+3    |36.3
+4    |36.3
+5    |36.3
+6    |36.3
+7    |36.3
+1    |36.3
+2    |36.3
+(10 rows)
+
+-- Downgrade to GPDB4 (AO version 2).
+--start_ignore
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'ao_upgrade_test'::regclass), 2::smallint, false);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_upgrade_test'::regclass), 2::smallint, true);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_rle_upgrade_test'::regclass), 2::smallint, true);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+--end_ignore
+
+-- Scan test. The numerics should be fixed again.
+SELECT * FROM ao_upgrade_test;
+rowid|n            
+-----+-------------
+8    |6227020800   
+9    |87178291200  
+10   |1307674368000
+1    |720          
+2    |5040         
+3    |40320        
+4    |362880       
+5    |3628800      
+6    |39916800     
+7    |479001600    
+(10 rows)
+SELECT * FROM aocs_upgrade_test;
+rowid|n            
+-----+-------------
+1    |720          
+2    |5040         
+3    |40320        
+4    |362880       
+5    |3628800      
+6    |39916800     
+7    |479001600    
+8    |6227020800   
+9    |87178291200  
+10   |1307674368000
+(10 rows)
+SELECT * FROM aocs_rle_upgrade_test;
+rowid|n     
+-----+------
+1    |362880
+2    |362880
+3    |362880
+4    |362880
+5    |362880
+6    |362880
+7    |362880
+8    |362880
+9    |362880
+10   |362880
+(10 rows)
+
+-- Fetch test. To force fetches, we'll add bitmap indexes and disable sequential
+-- scan.
+CREATE INDEX ao_bitmap_index ON ao_upgrade_test USING bitmap(n);
+CREATE
+CREATE INDEX aocs_bitmap_index ON aocs_upgrade_test USING bitmap(n);
+CREATE
+CREATE INDEX aocs_rle_bitmap_index ON aocs_rle_upgrade_test USING bitmap(n);
+CREATE
+
+SET enable_seqscan TO off;
+SET
+
+-- Ensure we're using a bitmap scan for our tests. Upgrade note to developers:
+-- the only thing that this test needs to verify is that a fetch-based scan is
+-- in use. Other diffs are fine.
+EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                           
+-----------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                     
+  ->  Bitmap Append-Only Row-Oriented Scan on ao_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                            
+        ->  Bitmap Index Scan on ao_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                
+              Index Cond: n = 362880::numeric                                                        
+Settings:  enable_seqscan=off                                                                        
+Optimizer status: legacy query optimizer                                                             
+(7 rows)
+EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                                
+----------------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                          
+  ->  Bitmap Append-Only Column-Oriented Scan on aocs_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                                 
+        ->  Bitmap Index Scan on aocs_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                   
+              Index Cond: n = 362880::numeric                                                             
+Settings:  enable_seqscan=off                                                                             
+Optimizer status: legacy query optimizer                                                                  
+(7 rows)
+EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                                    
+--------------------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                              
+  ->  Bitmap Append-Only Column-Oriented Scan on aocs_rle_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                                     
+        ->  Bitmap Index Scan on aocs_rle_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                   
+              Index Cond: n = 362880::numeric                                                                 
+Settings:  enable_seqscan=off                                                                                 
+Optimizer status: legacy query optimizer                                                                      
+(7 rows)
+
+SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+(1 row)
+SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+(1 row)
+SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+(10 rows)
+
+RESET enable_seqscan;
+RESET
+
+DROP FUNCTION convert_to_v4(numeric);
+DROP
+DROP FUNCTION set_ao_formatversion(oid, smallint, bool);
+DROP

src/test/isolation2/output/ao_upgrade_optimizer.source

+-- convert_to_v4 takes a modern numeric and converts it into a GPDB4 numeric.
+CREATE OR REPLACE FUNCTION convert_to_v4(value numeric) RETURNS numeric AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'convertNumericToGPDB4' LANGUAGE C RETURNS NULL ON NULL INPUT;
+CREATE
+
+-- set_ao_formatversion forces an AO[CO] format to a specific version (the last
+-- argument is set to true for a column-oriented table, and false otherwise).
+CREATE OR REPLACE FUNCTION set_ao_formatversion(aosegrel oid, version smallint, isaocs bool) RETURNS bool AS '@abs_builddir@/isolation2_regress@DLSUFFIX@', 'setAOFormatVersion' LANGUAGE C RETURNS NULL ON NULL INPUT;
+CREATE
+
+DROP TABLE IF EXISTS ao_upgrade_test;
+DROP
+DROP TABLE IF EXISTS aocs_upgrade_test;
+DROP
+
+CREATE TABLE ao_upgrade_test (rowid int, n numeric) WITH (appendonly=true);
+CREATE
+CREATE TABLE aocs_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column);
+CREATE
+CREATE TABLE aocs_rle_upgrade_test (rowid int, n numeric) WITH (appendonly=true, orientation=column, compresstype=RLE_TYPE);
+CREATE
+
+INSERT INTO ao_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+INSERT 10
+INSERT INTO aocs_upgrade_test (SELECT a, convert_to_v4((a + 5) !) FROM generate_series(1, 10) a);
+INSERT 10
+
+-- For the RLE test case, insert a bunch of identical numerics so they will be
+-- run-length compressed.
+INSERT INTO aocs_rle_upgrade_test (SELECT a, convert_to_v4(9 !) FROM generate_series(1, 10) a);
+INSERT 10
+
+-- Check to make sure convert_to_v4() does what we expect; without setting the
+-- AO format version back to GPDB4, the numerics we inserted should be mangled.
+SELECT * FROM ao_upgrade_test;
+rowid|n     
+-----+------
+3    |4.0   
+4    |36.3  
+5    |362.9 
+6    |3991.7
+7    |4.79  
+8    |62.27 
+9    |871.78
+10   |1.308 
+1    |720   
+2    |5040  
+(10 rows)
+SELECT * FROM aocs_upgrade_test;
+rowid|n     
+-----+------
+3    |4.0   
+4    |36.3  
+5    |362.9 
+6    |3991.7
+7    |4.79  
+8    |62.27 
+9    |871.78
+10   |1.308 
+1    |720   
+2    |5040  
+(10 rows)
+SELECT * FROM aocs_rle_upgrade_test;
+rowid|n   
+-----+----
+8    |36.3
+9    |36.3
+10   |36.3
+3    |36.3
+4    |36.3
+5    |36.3
+6    |36.3
+7    |36.3
+1    |36.3
+2    |36.3
+(10 rows)
+
+-- Downgrade to GPDB4 (AO version 2).
+--start_ignore
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'ao_upgrade_test'::regclass), 2::smallint, false);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_upgrade_test'::regclass), 2::smallint, true);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+*U: SELECT set_ao_formatversion( (SELECT segrelid FROM pg_appendonly WHERE relid = 'aocs_rle_upgrade_test'::regclass), 2::smallint, true);
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+
+set_ao_formatversion
+--------------------
+t                   
+(1 row)
+--end_ignore
+
+-- Scan test. The numerics should be fixed again.
+SELECT * FROM ao_upgrade_test;
+rowid|n            
+-----+-------------
+8    |6227020800   
+9    |87178291200  
+10   |1307674368000
+1    |720          
+2    |5040         
+3    |40320        
+4    |362880       
+5    |3628800      
+6    |39916800     
+7    |479001600    
+(10 rows)
+SELECT * FROM aocs_upgrade_test;
+rowid|n            
+-----+-------------
+1    |720          
+2    |5040         
+3    |40320        
+4    |362880       
+5    |3628800      
+6    |39916800     
+7    |479001600    
+8    |6227020800   
+9    |87178291200  
+10   |1307674368000
+(10 rows)
+SELECT * FROM aocs_rle_upgrade_test;
+rowid|n     
+-----+------
+1    |362880
+2    |362880
+3    |362880
+4    |362880
+5    |362880
+6    |362880
+7    |362880
+8    |362880
+9    |362880
+10   |362880
+(10 rows)
+
+-- Fetch test. To force fetches, we'll add bitmap indexes and disable sequential
+-- scan.
+CREATE INDEX ao_bitmap_index ON ao_upgrade_test USING bitmap(n);
+CREATE
+CREATE INDEX aocs_bitmap_index ON aocs_upgrade_test USING bitmap(n);
+CREATE
+CREATE INDEX aocs_rle_bitmap_index ON aocs_rle_upgrade_test USING bitmap(n);
+CREATE
+
+SET enable_seqscan TO off;
+SET
+
+-- Ensure we're using a bitmap scan for our tests. Upgrade note to developers:
+-- the only thing that this test needs to verify is that a fetch-based scan is
+-- in use. Other diffs are fine.
+EXPLAIN SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                           
+-----------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                     
+  ->  Bitmap Table Scan on ao_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                            
+        ->  Bitmap Index Scan on ao_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                
+              Index Cond: n = 362880::numeric                                                        
+Settings:  enable_seqscan=off                                                                        
+Optimizer status: legacy query optimizer                                                             
+(7 rows)
+EXPLAIN SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                                
+----------------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                          
+  ->  Bitmap Table Scan on aocs_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                                 
+        ->  Bitmap Index Scan on aocs_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                   
+              Index Cond: n = 362880::numeric                                                             
+Settings:  enable_seqscan=off                                                                             
+Optimizer status: legacy query optimizer                                                                  
+(7 rows)
+EXPLAIN SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+QUERY PLAN                                                                                                    
+--------------------------------------------------------------------------------------------------------------
+Gather Motion 3:1  (slice1; segments: 3)  (cost=1000.36..1100.37 rows=1 width=9)                              
+  ->  Bitmap Table Scan on aocs_rle_upgrade_test  (cost=1000.36..1100.37 rows=1 width=9)
+        Recheck Cond: n = 362880::numeric                                                                     
+        ->  Bitmap Index Scan on aocs_rle_bitmap_index  (cost=0.00..1000.36 rows=1 width=0)                   
+              Index Cond: n = 362880::numeric                                                                 
+Settings:  enable_seqscan=off                                                                                 
+Optimizer status: legacy query optimizer                                                                      
+(7 rows)
+
+SELECT n FROM ao_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+(1 row)
+SELECT n FROM aocs_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+(1 row)
+SELECT n FROM aocs_rle_upgrade_test WHERE n = (9 !);
+n     
+------
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+362880
+(10 rows)
+
+RESET enable_seqscan;
+RESET
+
+DROP FUNCTION convert_to_v4(numeric);
+DROP
+DROP FUNCTION set_ao_formatversion(oid, smallint, bool);
+DROP