1. full functional WAL for GiST

2. improve vacuum for gist
   - use FSM
   - full vacuum:
      - reforms parent tuple if it's needed
        ( tuples was deleted on child page or parent tuple remains invalid
          after crash recovery )
      - truncate index file if possible
3. fixes bugs and mistakes

src/backend/access/gist/Makefile

@@ -4,7 +4,7 @@
 #    Makefile for access/gist
 #
 # IDENTIFICATION
-#    $PostgreSQL: pgsql/src/backend/access/gist/Makefile,v 1.13 2005/06/14 11:45:13 teodor Exp $
+#    $PostgreSQL: pgsql/src/backend/access/gist/Makefile,v 1.14 2005/06/20 10:29:36 teodor Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -12,7 +12,7 @@ subdir = src/backend/access/gist
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = gist.o gistutil.o gistxlog.o gistget.o gistscan.o
+OBJS = gist.o gistutil.o gistxlog.o gistvacuum.o gistget.o gistscan.o
 
 all: SUBSYS.o
 

src/backend/access/gist/gistget.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.48 2005/06/14 11:45:13 teodor Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/gist/gistget.c,v 1.49 2005/06/20 10:29:36 teodor Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -112,7 +112,6 @@ gistnext(IndexScanDesc scan, ScanDirection dir)
 {
 	Page		p;
 	OffsetNumber n;
-	GISTPageOpaque po;
 	GISTScanOpaque so;
 	GISTSTACK  *stk;
 	IndexTuple	it;
@@ -127,7 +126,6 @@ gistnext(IndexScanDesc scan, ScanDirection dir)
 	}
 
 	p = BufferGetPage(so->curbuf);
-	po = (GISTPageOpaque) PageGetSpecialPointer(p);
 
 	if (ItemPointerIsValid(&scan->currentItemData) == false)
 	{
@@ -169,7 +167,6 @@ gistnext(IndexScanDesc scan, ScanDirection dir)
 			so->curbuf = ReleaseAndReadBuffer(so->curbuf, scan->indexRelation,
 											  stk->block);
 			p = BufferGetPage(so->curbuf);
-			po = (GISTPageOpaque) PageGetSpecialPointer(p);
 
 			if (ScanDirectionIsBackward(dir))
 				n = OffsetNumberPrev(stk->offset);
@@ -182,7 +179,7 @@ gistnext(IndexScanDesc scan, ScanDirection dir)
 			continue;
 		}
 
-		if (po->flags & F_LEAF)
+		if (GistPageIsLeaf(p))
 		{
 			/*
 			 * We've found a matching index entry in a leaf page, so
@@ -219,7 +216,6 @@ gistnext(IndexScanDesc scan, ScanDirection dir)
 			so->curbuf = ReleaseAndReadBuffer(so->curbuf, scan->indexRelation,
 											  child_block);
 			p = BufferGetPage(so->curbuf);
-			po = (GISTPageOpaque) PageGetSpecialPointer(p);
 
 			if (ScanDirectionIsBackward(dir))
 				n = PageGetMaxOffsetNumber(p);
@@ -256,6 +252,12 @@ gistindex_keytest(IndexTuple tuple,
 
 	IncrIndexProcessed();
 
+	/*
+         * Tuple doesn't restore after crash recovery because of inclomplete insert 
+         */
+	if ( !GistPageIsLeaf(p) && GistTupleIsInvalid(tuple) ) 
+		return true;
+
 	while (keySize > 0)
 	{
 		Datum		datum;
@@ -317,7 +319,6 @@ gistfindnext(IndexScanDesc scan, OffsetNumber n, ScanDirection dir)
 {
 	OffsetNumber	maxoff;
 	IndexTuple		it;
-	GISTPageOpaque	po;
 	GISTScanOpaque	so;
 	MemoryContext	oldcxt;
 	Page			p;
@@ -325,7 +326,6 @@ gistfindnext(IndexScanDesc scan, OffsetNumber n, ScanDirection dir)
 	so = (GISTScanOpaque) scan->opaque;
 	p = BufferGetPage(so->curbuf);
 	maxoff = PageGetMaxOffsetNumber(p);
-	po = (GISTPageOpaque) PageGetSpecialPointer(p);
 
 	/*
 	 * Make sure we're in a short-lived memory context when we invoke

src/backend/access/gist/gistutil.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *          $PostgreSQL: pgsql/src/backend/access/gist/gistutil.c,v 1.1 2005/06/14 11:45:13 teodor Exp $
+ *          $PostgreSQL: pgsql/src/backend/access/gist/gistutil.c,v 1.2 2005/06/20 10:29:36 teodor Exp $
  *-------------------------------------------------------------------------
  */
 #include "postgres.h"
@@ -19,6 +19,7 @@
 #include "access/heapam.h"
 #include "catalog/index.h"
 #include "miscadmin.h"
+#include "storage/freespace.h"
 
 /* group flags ( in gistadjsubkey ) */
 #define LEFT_ADDED      0x01
@@ -132,9 +133,14 @@ gistunion(Relation r, IndexTuple *itvec, int len, GISTSTATE *giststate)
 	GistEntryVector *evec;
 	int			i;
 	GISTENTRY	centry[INDEX_MAX_KEYS];
+	IndexTuple 	res;
 
 	evec = (GistEntryVector *) palloc(((len == 1) ? 2 : len) * sizeof(GISTENTRY) + GEVHDRSZ);
 
+	for(i = 0; i<len; i++) 
+		if ( GistTupleIsInvalid( itvec[i] ) )
+			return gist_form_invalid_tuple( InvalidBlockNumber );	
+
 	for (i = 0; i < r->rd_att->natts; i++)
 	{
 		Datum		datum;
@@ -191,7 +197,9 @@ gistunion(Relation r, IndexTuple *itvec, int len, GISTSTATE *giststate)
 		}
 	}
 
-	return index_form_tuple(giststate->tupdesc, attr, isnull);
+	res = index_form_tuple(giststate->tupdesc, attr, isnull);
+	GistTupleSetValid( res );
+	return res;
 }
 
 
@@ -215,11 +223,15 @@ gistgetadjusted(Relation r, IndexTuple oldtup, IndexTuple addtup, GISTSTATE *gis
 	IndexTuple	newtup = NULL;
 	int			i;
 
+	if ( GistTupleIsInvalid(oldtup) || GistTupleIsInvalid(addtup) )
+		return gist_form_invalid_tuple( ItemPointerGetBlockNumber( &(oldtup->t_tid) ) ); 
+ 
 	evec = palloc(2 * sizeof(GISTENTRY) + GEVHDRSZ);
 	evec->n = 2;
 	ev0p = &(evec->vector[0]);
 	ev1p = &(evec->vector[1]);
 
+
 	gistDeCompressAtt(giststate, r, oldtup, NULL,
 					  (OffsetNumber) 0, oldatt, oldisnull);
 
@@ -283,7 +295,7 @@ gistgetadjusted(Relation r, IndexTuple oldtup, IndexTuple addtup, GISTSTATE *gis
 }
 
 void
-gistunionsubkey(Relation r, GISTSTATE *giststate, IndexTuple *itvec, GIST_SPLITVEC *spl)
+gistunionsubkey(Relation r, GISTSTATE *giststate, IndexTuple *itvec, GIST_SPLITVEC *spl, bool isall)
 {
 	int lr;
 
@@ -314,9 +326,9 @@ gistunionsubkey(Relation r, GISTSTATE *giststate, IndexTuple *itvec, GIST_SPLITV
 			isnull = spl->spl_risnull;
 		}
 
-		evec = palloc(((len == 1) ? 2 : len) * sizeof(GISTENTRY) + GEVHDRSZ);
+		evec = palloc(((len < 2) ? 2 : len) * sizeof(GISTENTRY) + GEVHDRSZ);
 
-		for (i = 1; i < r->rd_att->natts; i++)
+		for (i = (isall) ? 0 : 1; i < r->rd_att->natts; i++)
 		{
 			int			j;
 			Datum		datum;
@@ -448,7 +460,7 @@ gistfindgroup(GISTSTATE *giststate, GISTENTRY *valvec, GIST_SPLITVEC *spl)
 void
 gistadjsubkey(Relation r,
 			  IndexTuple *itup, /* contains compressed entry */
-			  int *len,
+			  int len,
 			  GIST_SPLITVEC *v,
 			  GISTSTATE *giststate)
 {
@@ -501,7 +513,7 @@ gistadjsubkey(Relation r,
 	ev1p = &(evec->vector[1]);
 
 	/* add equivalent tuple */
-	for (i = 0; i < *len; i++)
+	for (i = 0; i < len; i++)
 	{
 		Datum		datum;
 
@@ -617,7 +629,7 @@ gistchoose(Relation r, Page p, IndexTuple it,	/* it has compressed entry */
 
 	maxoff = PageGetMaxOffsetNumber(p);
 	*which_grow = -1.0;
-	which = -1;
+	which = InvalidOffsetNumber;
 	sum_grow = 1;
 	gistDeCompressAtt(giststate, r,
 					  it, NULL, (OffsetNumber) 0,
@@ -627,6 +639,12 @@ gistchoose(Relation r, Page p, IndexTuple it,	/* it has compressed entry */
 	{
 		int			j;
 		IndexTuple	itup = (IndexTuple) PageGetItem(p, PageGetItemId(p, i));
+		
+		if ( !GistPageIsLeaf(p) && GistTupleIsInvalid(itup) ) {
+			elog(LOG, "It's desirable to vacuum or reindex GiST index '%s' due to crash recovery", 
+				RelationGetRelationName(r));
+			continue; 
+		}
 
 		sum_grow = 0;
 		for (j = 0; j < r->rd_att->natts; j++)
@@ -660,6 +678,9 @@ gistchoose(Relation r, Page p, IndexTuple it,	/* it has compressed entry */
 		}
 	}
 
+	if ( which == InvalidOffsetNumber )
+		which = FirstOffsetNumber;
+
 	return which;
 }
 
@@ -721,6 +742,7 @@ gistFormTuple(GISTSTATE *giststate, Relation r,
 	GISTENTRY	centry[INDEX_MAX_KEYS];
 	Datum		compatt[INDEX_MAX_KEYS];
 	int			i;
+	IndexTuple	res;
 
 	for (i = 0; i < r->rd_att->natts; i++)
 	{
@@ -735,7 +757,9 @@ gistFormTuple(GISTSTATE *giststate, Relation r,
 		}
 	}
 
-	return index_form_tuple(giststate->tupdesc, compatt, isnull);
+	res = index_form_tuple(giststate->tupdesc, compatt, isnull);
+	GistTupleSetValid(res);
+	return res;
 }
 
 void
@@ -783,3 +807,79 @@ GISTInitBuffer(Buffer b, uint32 f)
 	opaque->flags = f;
 }
 
+void
+gistUserPicksplit(Relation r, GistEntryVector *entryvec, GIST_SPLITVEC *v, 
+		IndexTuple *itup, int len, GISTSTATE *giststate) {
+	/*
+	 * now let the user-defined picksplit function set up the split
+	 * vector; in entryvec have no null value!!
+	 */
+	FunctionCall2(&giststate->picksplitFn[0],
+				  PointerGetDatum(entryvec),
+				  PointerGetDatum(v));
+
+	/* compatibility with old code */
+	if (v->spl_left[v->spl_nleft - 1] == InvalidOffsetNumber)
+		v->spl_left[v->spl_nleft - 1] = (OffsetNumber) (entryvec->n - 1);
+	if (v->spl_right[v->spl_nright - 1] == InvalidOffsetNumber)
+		v->spl_right[v->spl_nright - 1] = (OffsetNumber) (entryvec->n - 1);
+
+	v->spl_lattr[0] = v->spl_ldatum;
+	v->spl_rattr[0] = v->spl_rdatum;
+	v->spl_lisnull[0] = false;
+	v->spl_risnull[0] = false;
+
+	/*
+	 * if index is multikey, then we must to try get smaller bounding box
+	 * for subkey(s)
+	 */
+	if (r->rd_att->natts > 1)
+	{
+		int			MaxGrpId;
+
+		v->spl_idgrp = (int *) palloc0(sizeof(int) * entryvec->n);
+		v->spl_grpflag = (char *) palloc0(sizeof(char) * entryvec->n);
+		v->spl_ngrp = (int *) palloc(sizeof(int) * entryvec->n);
+
+		MaxGrpId = gistfindgroup(giststate, entryvec->vector, v);
+
+		/* form union of sub keys for each page (l,p) */
+		gistunionsubkey(r, giststate, itup, v, false);
+
+		/*
+		 * if possible, we insert equivalent tuples with control by
+		 * penalty for a subkey(s)
+		 */
+		if (MaxGrpId > 1)
+			gistadjsubkey(r, itup, len, v, giststate);
+	}
+}
+
+Buffer  
+gistReadBuffer(Relation r, BlockNumber blkno) {
+	Buffer buffer = InvalidBuffer;
+
+	if ( blkno != P_NEW ) {
+		buffer = ReadBuffer(r, blkno);
+	} else {
+		Page page;
+
+		while(true) {
+			blkno = GetFreeIndexPage(&r->rd_node);
+			if (blkno == InvalidBlockNumber)
+				break;
+
+			buffer = ReadBuffer(r, blkno);
+			page = BufferGetPage(buffer);
+			if ( GistPageIsDeleted( page ) ) {
+				GistPageSetNonDeleted( page );
+				return buffer;
+			}
+			ReleaseBuffer( buffer );
+		}
+
+		buffer = ReadBuffer(r, P_NEW); 
+	}
+	
+	return buffer;
+}

src/backend/access/gist/gistvacuum.c

+/*-------------------------------------------------------------------------
+ *
+ * gistvacuum.c
+ *	  interface routines for the postgres GiST index access method.
+ *
+ *
+ * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * IDENTIFICATION
+ *	  $PostgreSQL: pgsql/src/backend/access/gist/gistvacuum.c,v 1.1 2005/06/20 10:29:36 teodor Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/genam.h"
+#include "access/gist_private.h"
+#include "access/gistscan.h"
+#include "access/heapam.h"
+#include "catalog/index.h"
+#include "commands/vacuum.h"
+#include "miscadmin.h"
+#include "utils/memutils.h"
+#include "storage/freespace.h"
+#include "storage/smgr.h"
+
+/* filled by gistbulkdelete, cleared by gistvacuumpcleanup */ 
+static bool needFullVacuum = false; 
+
+
+typedef struct {
+	GISTSTATE	giststate;
+	Relation	index;
+	MemoryContext	opCtx;
+	IndexBulkDeleteResult	*result;
+
+	/* path to root */
+	BlockNumber	*path;
+	int		pathlen;
+	int		curpathlen;
+} GistVacuum;
+
+static void
+shiftPath(GistVacuum *gv, BlockNumber blkno) {
+	if ( gv->pathlen == 0 ) {
+		gv->pathlen = 8;
+		gv->path = (BlockNumber*) palloc( MAXALIGN(sizeof(BlockNumber)*gv->pathlen) );
+	} else if ( gv->pathlen == gv->curpathlen ) {
+		gv->pathlen *= 2;
+		gv->path = (BlockNumber*) repalloc( gv->path, MAXALIGN(sizeof(BlockNumber)*gv->pathlen) );
+	}
+
+	if ( gv->curpathlen )
+		memmove( gv->path+1, gv->path, sizeof(BlockNumber)*gv->curpathlen ); 
+	gv->curpathlen++;
+	gv->path[0] = blkno;
+}
+
+static void
+unshiftPath(GistVacuum *gv) {
+	gv->curpathlen--;
+	if ( gv->curpathlen )
+		memmove( gv->path, gv->path+1, sizeof(BlockNumber)*gv->curpathlen );
+} 
+
+typedef struct {
+	IndexTuple	*itup;
+	int		ituplen;
+	bool		emptypage;
+} ArrayTuple;
+
+
+static ArrayTuple
+gistVacuumUpdate( GistVacuum *gv, BlockNumber blkno, bool needunion ) {
+	ArrayTuple	res = {NULL, 0, false};
+	Buffer		buffer;
+	Page		page;
+	OffsetNumber 	i, maxoff;
+	ItemId		iid;
+	int 		lenaddon=4, curlenaddon=0, ntodelete=0;
+	IndexTuple	idxtuple, *addon=NULL;
+	bool		needwrite=false;
+	OffsetNumber    *todelete=NULL;
+	ItemPointerData	*completed=NULL;
+	int 		ncompleted=0, lencompleted=16;
+
+	buffer = ReadBuffer(gv->index, blkno);
+	page = (Page) BufferGetPage(buffer);
+	maxoff = PageGetMaxOffsetNumber(page);
+
+
+	if ( GistPageIsLeaf(page) ) {
+		if ( GistTuplesDeleted(page) ) {
+			needunion = needwrite = true;
+			GistClearTuplesDeleted(page);
+		}
+	} else {
+		todelete = (OffsetNumber*)palloc( MAXALIGN(sizeof(OffsetNumber)*(maxoff+1)) );
+		completed = (ItemPointerData*)palloc( sizeof(ItemPointerData)*lencompleted );
+		addon=(IndexTuple*)palloc(sizeof(IndexTuple)*lenaddon);
+
+		shiftPath(gv, blkno);
+		for (i = FirstOffsetNumber; i <= maxoff; i = OffsetNumberNext(i)) {
+			ArrayTuple chldtuple;
+			bool needchildunion;
+
+			iid = PageGetItemId(page, i);
+			idxtuple = (IndexTuple) PageGetItem(page, iid);
+			needchildunion = (GistTupleIsInvalid(idxtuple)) ? true : false;
+		
+			if ( needchildunion ) 
+				elog(DEBUG2,"gistVacuumUpdate: Need union for block %u", ItemPointerGetBlockNumber(&(idxtuple->t_tid)));
+	
+			chldtuple = gistVacuumUpdate( gv, ItemPointerGetBlockNumber(&(idxtuple->t_tid)),
+				needchildunion );
+			if ( chldtuple.ituplen || chldtuple.emptypage ) {
+				/* adjust any scans that will be affected by this deletion */
+				gistadjscans(gv->index, GISTOP_DEL, blkno, i);
+				PageIndexTupleDelete(page, i);
+				todelete[ ntodelete++ ] = i;
+				i--; maxoff--;
+				needwrite=needunion=true;
+
+				if ( chldtuple.ituplen ) {
+					while( curlenaddon + chldtuple.ituplen >= lenaddon ) {
+						lenaddon*=2;
+						addon=(IndexTuple*)repalloc( addon, sizeof(IndexTuple)*lenaddon );
+					}
+
+					memcpy( addon + curlenaddon, chldtuple.itup, chldtuple.ituplen * sizeof(IndexTuple) );
+
+					curlenaddon += chldtuple.ituplen;
+
+					if ( chldtuple.ituplen > 1 ) {
+						/* child was splitted, so we need mark completion insert(split) */
+						int j;
+
+						while( ncompleted + chldtuple.ituplen > lencompleted ) {
+							lencompleted*=2;
+							completed = (ItemPointerData*)repalloc(completed, sizeof(ItemPointerData) * lencompleted);
+						} 
+						for(j=0;j<chldtuple.ituplen;j++) {
+							ItemPointerCopy( &(chldtuple.itup[j]->t_tid), completed + ncompleted ); 
+							ncompleted++; 
+						}
+					}
+					pfree( chldtuple.itup );
+				}
+			}
+		}
+
+		if ( curlenaddon ) {
+			/* insert updated tuples */
+			if (gistnospace(page, addon, curlenaddon)) {
+				/* there is no space on page to insert tuples */
+				IndexTuple	*vec;
+				SplitedPageLayout       *dist=NULL,*ptr;
+				int i;
+				MemoryContext oldCtx = MemoryContextSwitchTo(gv->opCtx); 
+
+				vec = gistextractbuffer(buffer, &(res.ituplen));
+				vec = gistjoinvector(vec, &(res.ituplen), addon, curlenaddon);
+				res.itup = gistSplit(gv->index, buffer, vec, &(res.ituplen), &dist, &(gv->giststate)); 
+				MemoryContextSwitchTo(oldCtx);
+
+				vec = (IndexTuple*)palloc( sizeof(IndexTuple) * res.ituplen );
+				for(i=0;i<res.ituplen;i++) {
+					vec[i] = (IndexTuple)palloc( IndexTupleSize(res.itup[i]) );
+					memcpy( vec[i], res.itup[i], IndexTupleSize(res.itup[i]) );
+				}
+				res.itup = vec; 
+
+				if ( !gv->index->rd_istemp ) {
+					XLogRecPtr              recptr;
+					XLogRecData             *rdata;
+					ItemPointerData		key; /* set key for incomplete insert */
+
+					ItemPointerSet(&key, blkno, TUPLE_IS_VALID);
+	
+					oldCtx = MemoryContextSwitchTo(gv->opCtx);
+
+					/* path is need to recovery because there is new pages, in a case of
+					   crash it's needed to add inner tuple pointers on parent page */ 
+					rdata = formSplitRdata(gv->index->rd_node, blkno,
+						todelete, ntodelete, addon, curlenaddon,
+						&key, gv->path, gv->curpathlen, dist);
+
+					MemoryContextSwitchTo(oldCtx);
+					
+					START_CRIT_SECTION();
+			
+					recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_PAGE_SPLIT, rdata);
+					ptr = dist;
+					while(ptr) {
+						PageSetLSN(BufferGetPage(ptr->buffer), recptr);
+						PageSetTLI(BufferGetPage(ptr->buffer), ThisTimeLineID);
+						ptr=ptr->next;
+					}
+
+					END_CRIT_SECTION();
+				
+				}
+
+				ptr = dist;
+				while(ptr) {
+					WriteBuffer(ptr->buffer);
+					ptr=ptr->next;
+				}
+
+				if ( blkno == GIST_ROOT_BLKNO ) { 
+					ItemPointerData		key; /* set key for incomplete insert */
+
+					ItemPointerSet(&key, blkno, TUPLE_IS_VALID);
+
+					oldCtx = MemoryContextSwitchTo(gv->opCtx);
+					gistnewroot(gv->index, res.itup, res.ituplen, &key);
+					MemoryContextSwitchTo(oldCtx);
+				}
+
+				needwrite=false;
+ 
+				MemoryContextReset(gv->opCtx);
+
+				needunion = false; /* gistSplit already forms unions */
+			} else {
+				OffsetNumber off = (PageIsEmpty(page)) ?
+					FirstOffsetNumber
+					:
+					OffsetNumberNext(PageGetMaxOffsetNumber(page));
+
+				/* enough free space */
+				gistfillbuffer(gv->index, page, addon, curlenaddon, off); 
+			} 
+		}
+		unshiftPath(gv);
+	}
+
+	if ( needunion ) {
+		/* forms union for page  or check empty*/
+		if ( PageIsEmpty(page) ) {
+			if ( blkno == GIST_ROOT_BLKNO ) {
+				needwrite=true;
+				GistPageSetLeaf( page );
+			} else {
+				needwrite=true;
+				res.emptypage=true;
+				GistPageSetDeleted( page );
+				gv->result->pages_deleted++;
+			}
+		} else {
+			IndexTuple	*vec, tmp;
+			int		veclen=0;
+			MemoryContext oldCtx = MemoryContextSwitchTo(gv->opCtx);
+ 
+			vec = gistextractbuffer(buffer, &veclen);
+			tmp  = gistunion(gv->index, vec, veclen, &(gv->giststate));
+			MemoryContextSwitchTo(oldCtx);
+
+			res.itup=(IndexTuple*)palloc( sizeof(IndexTuple) );
+			res.ituplen = 1;
+			res.itup[0] = (IndexTuple)palloc( IndexTupleSize(tmp) );
+			memcpy( res.itup[0], tmp, IndexTupleSize(tmp) );
+
+			ItemPointerSetBlockNumber(&(res.itup[0]->t_tid), blkno);
+			GistTupleSetValid( res.itup[0] );	 
+		
+			MemoryContextReset(gv->opCtx);
+		}
+	}
+
+	if ( needwrite ) {
+		if ( !gv->index->rd_istemp ) {
+			XLogRecData *rdata;
+			XLogRecPtr	recptr;
+			MemoryContext oldCtx = MemoryContextSwitchTo(gv->opCtx);
+
+			/* In a vacuum, it's not need to push path, because
+			   there is no new inserted keys */
+ 			rdata = formUpdateRdata(gv->index->rd_node, blkno, todelete, ntodelete, 
+				res.emptypage, addon, curlenaddon, NULL, NULL, 0);
+			MemoryContextSwitchTo(oldCtx);
+		
+	
+			START_CRIT_SECTION();
+			recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_ENTRY_UPDATE, rdata);
+			PageSetLSN(page, recptr);
+			PageSetTLI(page, ThisTimeLineID);
+			END_CRIT_SECTION();
+			MemoryContextReset(gv->opCtx);
+		}
+		WriteBuffer( buffer );
+	} else
+		ReleaseBuffer( buffer );
+
+	if ( ncompleted && !gv->index->rd_istemp )
+		gistxlogInsertCompletion( gv->index->rd_node, completed, ncompleted );
+
+	for(i=0;i<curlenaddon;i++)
+		pfree( addon[i] );
+	if (addon) pfree(addon);
+	if (todelete) pfree(todelete); 
+	if (completed) pfree(completed); 
+	return res;
+}
+
+/*
+ * For usial vacuum just update FSM, for full vacuum
+ * reforms parent tuples if some of childs was deleted or changed,
+ * update invalid tuples (they can exsist from last crash recovery only),
+ * tries to get smaller index
+ */
+
+Datum
+gistvacuumcleanup(PG_FUNCTION_ARGS) {
+	Relation	rel = (Relation) PG_GETARG_POINTER(0);
+	IndexVacuumCleanupInfo *info = (IndexVacuumCleanupInfo *) PG_GETARG_POINTER(1);
+	IndexBulkDeleteResult *stats = (IndexBulkDeleteResult *) PG_GETARG_POINTER(2);
+	BlockNumber npages, blkno;
+	BlockNumber nFreePages, *freePages, maxFreePages;
+	BlockNumber lastBlock = GIST_ROOT_BLKNO, lastFilledBlock = GIST_ROOT_BLKNO;	
+
+	/* LockRelation(rel, AccessExclusiveLock); */
+
+	/* gistVacuumUpdate may cause hard work */
+	if ( info->vacuum_full ) {
+		GistVacuum	gv;
+		ArrayTuple	res;
+
+		gv.index = rel;
+		initGISTstate(&(gv.giststate), rel);
+		gv.opCtx = createTempGistContext();
+		gv.result = stats;
+
+		gv.path=NULL;
+		gv.pathlen = gv.curpathlen = 0;
+
+		/* walk through the entire index for update tuples */
+		res = gistVacuumUpdate( &gv, GIST_ROOT_BLKNO, false );
+        	/* cleanup */
+		if (res.itup) {
+			int i;
+			for(i=0;i<res.ituplen;i++)
+				pfree( res.itup[i] );
+			pfree( res.itup );
+		}
+		if ( gv.path )
+			pfree( gv.path );
+        	freeGISTstate(&(gv.giststate));
+        	MemoryContextDelete(gv.opCtx);
+	} else if (needFullVacuum) {
+		elog(NOTICE,"It's desirable to vacuum full or reindex GiST index '%s' due to crash recovery", 
+			RelationGetRelationName(rel));
+	}
+
+	needFullVacuum = false;
+
+	/* try to find deleted pages */
+	npages = RelationGetNumberOfBlocks(rel);
+	maxFreePages = RelationGetNumberOfBlocks(rel);
+	if ( maxFreePages > MaxFSMPages )
+		maxFreePages = MaxFSMPages;
+	nFreePages = 0;
+	freePages = (BlockNumber*) palloc (sizeof(BlockNumber) * maxFreePages);
+	for(blkno=GIST_ROOT_BLKNO+1;blkno<npages;blkno++) {
+		Buffer	buffer = ReadBuffer(rel, blkno);
+		Page	page=(Page)BufferGetPage(buffer);
+
+		if ( GistPageIsDeleted(page) ) {
+			if (nFreePages < maxFreePages) {
+				freePages[ nFreePages ] = blkno;
+				nFreePages++;
+			}
+		} else
+			lastFilledBlock = blkno;
+		ReleaseBuffer(buffer);
+	}
+	lastBlock = npages-1;
+		
+	if ( nFreePages > 0 ) {
+		if ( info->vacuum_full ) { /* try to truncate index */
+			int i;
+			for(i=0;i<nFreePages;i++)
+				if ( freePages[i] >= lastFilledBlock ) {
+					nFreePages = i;
+					break;
+				}
+	
+			if ( lastBlock > lastFilledBlock )	
+				RelationTruncate( rel, lastFilledBlock+1 );
+			stats->pages_removed = lastBlock - lastFilledBlock;
+		}
+		
+		if ( nFreePages > 0 )
+			RecordIndexFreeSpace( &rel->rd_node, nFreePages, freePages );
+	}
+	pfree( freePages ); 
+
+	/* return statistics */
+	stats->pages_free = nFreePages;
+	stats->num_pages = RelationGetNumberOfBlocks(rel);
+
+	/* UnlockRelation(rel, AccessExclusiveLock); */
+
+	PG_RETURN_POINTER(stats);
+}
+
+typedef struct GistBDItem {
+	BlockNumber 	blkno;
+	struct GistBDItem *next; 
+} GistBDItem;
+
+/*
+ * Bulk deletion of all index entries pointing to a set of heap tuples and
+ * update invalid tuples after crash recovery.
+ * The set of target tuples is specified via a callback routine that tells
+ * whether any given heap tuple (identified by ItemPointer) is being deleted.
+ *
+ * Result: a palloc'd struct containing statistical info for VACUUM displays.
+ */
+Datum
+gistbulkdelete(PG_FUNCTION_ARGS) {
+	Relation	rel = (Relation) PG_GETARG_POINTER(0);
+	IndexBulkDeleteCallback callback = (IndexBulkDeleteCallback) PG_GETARG_POINTER(1);
+	void* callback_state = (void *) PG_GETARG_POINTER(2);
+	IndexBulkDeleteResult	*result = (IndexBulkDeleteResult*)palloc0(sizeof(IndexBulkDeleteResult));	
+	GistBDItem	*stack, *ptr;
+	MemoryContext opCtx = createTempGistContext();
+	
+	stack = (GistBDItem*) palloc(sizeof(GistBDItem));
+
+	stack->blkno = GIST_ROOT_BLKNO;
+	stack->next = NULL;
+	needFullVacuum = false;
+
+	while( stack ) {
+		Buffer buffer = ReadBuffer(rel, stack->blkno);
+		Page   page   = (Page) BufferGetPage(buffer);
+		OffsetNumber i, maxoff = PageGetMaxOffsetNumber(page);
+		IndexTuple	idxtuple;
+		ItemId		iid;
+		OffsetNumber *todelete = NULL;
+		int ntodelete = 0;	
+
+		if ( GistPageIsLeaf(page) ) {
+			ItemPointerData heapptr;
+
+			todelete = (OffsetNumber*)palloc( MAXALIGN(sizeof(OffsetNumber)*maxoff) );
+
+			for(i=FirstOffsetNumber;i<=maxoff;i=OffsetNumberNext(i)) {
+				iid = PageGetItemId(page, i);	
+				idxtuple = (IndexTuple) PageGetItem(page, iid);
+				heapptr = idxtuple->t_tid;
+
+				if ( callback(&heapptr, callback_state) ) {
+					gistadjscans(rel, GISTOP_DEL, stack->blkno, i);
+					PageIndexTupleDelete(page, i);
+					todelete[ ntodelete++ ] = i;
+					i--; maxoff--;
+					result->tuples_removed += 1;
+				} else 
+					result->num_index_tuples += 1;
+			}
+		} else {
+			for(i=FirstOffsetNumber;i<=maxoff;i=OffsetNumberNext(i)) {
+				iid = PageGetItemId(page, i);
+				idxtuple = (IndexTuple) PageGetItem(page, iid);
+
+				ptr = (GistBDItem*) palloc(sizeof(GistBDItem));
+				ptr->blkno = ItemPointerGetBlockNumber( &(idxtuple->t_tid) );
+				ptr->next = stack->next;
+				stack->next = ptr;
+
+				if ( GistTupleIsInvalid(idxtuple) )
+					needFullVacuum = true;
+			}
+		}
+
+		if ( ntodelete && todelete ) {
+			GistMarkTuplesDeleted(page);
+
+			if (!rel->rd_istemp ) {
+				XLogRecData *rdata;
+				XLogRecPtr      recptr;
+				MemoryContext oldCtx = MemoryContextSwitchTo(opCtx);
+
+				rdata = formUpdateRdata(rel->rd_node, stack->blkno, todelete, ntodelete,
+					false, NULL, 0, NULL, NULL, 0);
+				MemoryContextSwitchTo(oldCtx);
+
+				START_CRIT_SECTION();
+				recptr = XLogInsert(RM_GIST_ID, XLOG_GIST_ENTRY_UPDATE, rdata);
+				PageSetLSN(page, recptr);
+				PageSetTLI(page, ThisTimeLineID);
+				END_CRIT_SECTION();
+
+				MemoryContextReset(opCtx);
+			}
+
+			WriteBuffer( buffer );
+		} else
+			ReleaseBuffer( buffer );
+
+		if ( todelete )
+			pfree( todelete );
+
+		ptr = stack->next;
+		pfree( stack );
+		stack = ptr;
+	}
+
+	MemoryContextDelete( opCtx );
+
+	result->num_pages = RelationGetNumberOfBlocks(rel);
+
+
+	PG_RETURN_POINTER( result );
+}
+

src/include/access/gist.h

@@ -9,7 +9,7 @@
  * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/gist.h,v 1.46 2005/05/17 03:34:18 neilc Exp $
+ * $PostgreSQL: pgsql/src/include/access/gist.h,v 1.47 2005/06/20 10:29:36 teodor Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -36,6 +36,8 @@
  * Page opaque data in a GiST index page.
  */
 #define F_LEAF			(1 << 0)
+#define F_DELETED		(1 << 1)
+#define F_TUPLES_DELETED	(1 << 2)
 
 typedef struct GISTPageOpaqueData
 {
@@ -56,6 +58,7 @@ typedef struct GIST_SPLITVEC
 												 * spl_left */
 	int			spl_lattrsize[INDEX_MAX_KEYS];
 	bool		spl_lisnull[INDEX_MAX_KEYS];
+	bool		spl_leftvalid;
 
 	OffsetNumber *spl_right;	/* array of entries that go right */
 	int			spl_nright;		/* size of the array */
@@ -64,6 +67,7 @@ typedef struct GIST_SPLITVEC
 												 * spl_right */
 	int			spl_rattrsize[INDEX_MAX_KEYS];
 	bool		spl_risnull[INDEX_MAX_KEYS];
+	bool		spl_rightvalid;
 
 	int		   *spl_idgrp;
 	int		   *spl_ngrp;		/* number in each group */
@@ -86,7 +90,18 @@ typedef struct GISTENTRY
 	bool		leafkey;
 } GISTENTRY;
 
-#define GIST_LEAF(entry) (((GISTPageOpaque) PageGetSpecialPointer((entry)->page))->flags & F_LEAF)
+#define GistPageIsLeaf(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags & F_LEAF)
+#define GIST_LEAF(entry) (GistPageIsLeaf((entry)->page))
+#define GistPageSetLeaf(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags |= F_LEAF)
+#define GistPageSetNonLeaf(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags &= ~F_LEAF)
+
+#define GistPageIsDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags & F_DELETED)
+#define GistPageSetDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags |= F_DELETED)
+#define GistPageSetNonDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags &= ~F_DELETED)
+
+#define GistTuplesDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags & F_TUPLES_DELETED)
+#define GistMarkTuplesDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags |= F_TUPLES_DELETED)
+#define GistClearTuplesDeleted(page)	(((GISTPageOpaque) PageGetSpecialPointer(page))->flags &= ~F_TUPLES_DELETED)
 
 /*
  * Vector of GISTENTRY structs; user-defined methods union and pick

src/include/access/gist_private.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/gist_private.h,v 1.3 2005/06/14 11:45:14 teodor Exp $
+ * $PostgreSQL: pgsql/src/include/access/gist_private.h,v 1.4 2005/06/20 10:29:36 teodor Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -94,7 +94,6 @@ typedef struct {
 	int             ituplen; /* length of itup */
 	GISTInsertStack	*stack;
 	bool needInsertComplete;
-	bool xlog_mode;
 
 	/* pointer to heap tuple */
 	ItemPointerData	key;
@@ -142,19 +141,20 @@ typedef struct gistxlogEntryUpdate {
 	RelFileNode	node;
 	BlockNumber	blkno;
 
-	/* if todeleteoffnum!=InvalidOffsetNumber then delete it. */ 
-	OffsetNumber	todeleteoffnum;
+	uint16		ntodelete;
 	uint16		pathlen;
+	bool		isemptypage;	
 
 	/* 
-	 * It used to identify compliteness of insert.
+	 * It used to identify completeness of insert.
          * Sets to leaf itup 
          */ 
 	ItemPointerData	key;
 
 	/* follow:
-	 * 1. path to root (BlockNumber) 
-	 * 2. tuples to insert
+	 * 1. path to root (BlockNumber)
+	 * 2. todelete OffsetNumbers 
+	 * 3. tuples to insert
          */ 
 } gistxlogEntryUpdate;
 
@@ -163,18 +163,19 @@ typedef struct gistxlogEntryUpdate {
 typedef struct gistxlogPageSplit {
 	RelFileNode	node;
 	BlockNumber	origblkno; /*splitted page*/
-	OffsetNumber	todeleteoffnum;
+	uint16		ntodelete;
 	uint16		pathlen;
-	int		npage;
-	int		nitup;
+	uint16		npage;
+	uint16		nitup;
 
 	/* see comments on gistxlogEntryUpdate */
 	ItemPointerData	key;
  
 	/* follow:
 	 * 1. path to root (BlockNumber) 
-	 * 2. tuples to insert
-	 * 3. gistxlogPage and array of OffsetNumber per page
+	 * 2. todelete OffsetNumbers 
+	 * 3. tuples to insert
+	 * 4. gistxlogPage and array of OffsetNumber per page
          */ 
 } gistxlogPageSplit;
 
@@ -188,32 +189,65 @@ typedef struct gistxlogPage {
 
 typedef struct gistxlogInsertComplete {
 	RelFileNode	node;
-	ItemPointerData	key;
+	/* follows ItemPointerData key to clean */
 } gistxlogInsertComplete;
 
-#define XLOG_GIST_CREATE_INDEX	0x50
+#define	XLOG_GIST_CREATE_INDEX	0x50
+
+/*
+ * mark tuples on inner pages during recovery
+ */
+#define TUPLE_IS_VALID		0xffff
+#define TUPLE_IS_INVALID	0xfffe
+
+#define  GistTupleIsInvalid(itup)	( ItemPointerGetOffsetNumber( &((itup)->t_tid) ) == TUPLE_IS_INVALID )
+#define  GistTupleSetValid(itup)	ItemPointerSetOffsetNumber( &((itup)->t_tid), TUPLE_IS_VALID )
+#define  GistTupleSetInvalid(itup)	ItemPointerSetOffsetNumber( &((itup)->t_tid), TUPLE_IS_INVALID )
 
 /* gist.c */
 extern Datum gistbuild(PG_FUNCTION_ARGS);
 extern Datum gistinsert(PG_FUNCTION_ARGS);
-extern Datum gistbulkdelete(PG_FUNCTION_ARGS);
 extern MemoryContext createTempGistContext(void);
 extern void initGISTstate(GISTSTATE *giststate, Relation index);
 extern void freeGISTstate(GISTSTATE *giststate);
-extern void gistnewroot(Relation r, IndexTuple *itup, int len, ItemPointer key, bool xlog_mode);
+extern void gistnewroot(Relation r, IndexTuple *itup, int len, ItemPointer key);
 extern void gistmakedeal(GISTInsertState *state, GISTSTATE *giststate);
 
+typedef struct SplitedPageLayout {
+        gistxlogPage    block;
+        OffsetNumber    *list;
+        Buffer          buffer; /* to write after all proceed */
+
+        struct SplitedPageLayout *next;
+} SplitedPageLayout;
+
+IndexTuple * gistSplit(Relation r, Buffer buffer, IndexTuple *itup,
+                  int *len, SplitedPageLayout    **dist, GISTSTATE *giststate);
 /* gistxlog.c */
 extern void gist_redo(XLogRecPtr lsn, XLogRecord *record);
 extern void gist_desc(char *buf, uint8 xl_info, char *rec);
 extern void gist_xlog_startup(void);
 extern void gist_xlog_cleanup(void);
+extern IndexTuple gist_form_invalid_tuple(BlockNumber blkno);
+
+extern XLogRecData* formUpdateRdata(RelFileNode node, BlockNumber blkno,
+                OffsetNumber *todelete, int ntodelete, bool emptypage,
+                IndexTuple *itup, int ituplen, ItemPointer key,
+                BlockNumber *path, int pathlen);
+
+extern XLogRecData* formSplitRdata(RelFileNode node, BlockNumber blkno,
+                OffsetNumber *todelete, int ntodelete, 
+                IndexTuple *itup, int ituplen, ItemPointer key,
+                BlockNumber *path, int pathlen, SplitedPageLayout *dist );
+
+extern XLogRecPtr gistxlogInsertCompletion(RelFileNode node, ItemPointerData *keys, int len);
 
 /* gistget.c */
 extern Datum gistgettuple(PG_FUNCTION_ARGS);
 extern Datum gistgetmulti(PG_FUNCTION_ARGS);
 
 /* gistutil.c */
+extern	Buffer	gistReadBuffer(Relation r, BlockNumber blkno);
 extern OffsetNumber gistfillbuffer(Relation r, Page page, IndexTuple *itup,
                                 int len, OffsetNumber off);
 extern bool gistnospace(Page page, IndexTuple *itvec, int len);
@@ -230,7 +264,7 @@ extern IndexTuple gistgetadjusted(Relation r,
 extern int gistfindgroup(GISTSTATE *giststate,
                           GISTENTRY *valvec, GIST_SPLITVEC *spl);
 extern void gistadjsubkey(Relation r,
-                          IndexTuple *itup, int *len,
+                          IndexTuple *itup, int len,
                           GIST_SPLITVEC *v,
                           GISTSTATE *giststate);
 extern IndexTuple gistFormTuple(GISTSTATE *giststate,
@@ -247,10 +281,16 @@ extern void gistDeCompressAtt(GISTSTATE *giststate, Relation r,
                               IndexTuple tuple, Page p, OffsetNumber o,
                               GISTENTRY *attdata, bool *isnull);
 extern void gistunionsubkey(Relation r, GISTSTATE *giststate, 
-                            IndexTuple *itvec, GIST_SPLITVEC *spl);
+                            IndexTuple *itvec, GIST_SPLITVEC *spl, bool isall);
 extern void GISTInitBuffer(Buffer b, uint32 f);
 extern void gistdentryinit(GISTSTATE *giststate, int nkey, GISTENTRY *e,
 			   Datum k, Relation r, Page pg, OffsetNumber o,
 			   int b, bool l, bool isNull);
+void gistUserPicksplit(Relation r, GistEntryVector *entryvec, GIST_SPLITVEC *v,
+                IndexTuple *itup, int len, GISTSTATE *giststate);
+
+/* gistvacuum.c */
+extern Datum gistbulkdelete(PG_FUNCTION_ARGS);
+extern Datum gistvacuumcleanup(PG_FUNCTION_ARGS);
 
 #endif	/* GIST_PRIVATE_H */

src/include/catalog/catversion.h

@@ -37,7 +37,7 @@
  * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.278 2005/06/18 19:33:42 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.279 2005/06/20 10:29:37 teodor Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -53,6 +53,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	200506181
+#define CATALOG_VERSION_NO	200506201
 
 #endif

src/include/catalog/pg_am.h

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_am.h,v 1.34 2005/06/13 23:14:49 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_am.h,v 1.35 2005/06/20 10:29:37 teodor Exp $
  *
  * NOTES
  *		the genbki.sh script reads this file and generates .bki
@@ -112,7 +112,7 @@ DESCR("b-tree index access method");
 DATA(insert OID = 405 (  hash	1 1 0 f f f f t hashinsert hashbeginscan hashgettuple hashgetmulti hashrescan hashendscan hashmarkpos hashrestrpos hashbuild hashbulkdelete - hashcostestimate ));
 DESCR("hash index access method");
 #define HASH_AM_OID 405
-DATA(insert OID = 783 (  gist	100 7 0 f t f f f gistinsert gistbeginscan gistgettuple gistgetmulti gistrescan gistendscan gistmarkpos gistrestrpos gistbuild gistbulkdelete - gistcostestimate ));
+DATA(insert OID = 783 (  gist	100 7 0 f t f f f gistinsert gistbeginscan gistgettuple gistgetmulti gistrescan gistendscan gistmarkpos gistrestrpos gistbuild gistbulkdelete gistvacuumcleanup gistcostestimate ));
 DESCR("GiST index access method");
 #define GIST_AM_OID 783
 

src/include/catalog/pg_proc.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2005, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.368 2005/06/17 22:32:48 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_proc.h,v 1.369 2005/06/20 10:29:37 teodor Exp $
  *
  * NOTES
  *	  The script catalog/genbki.sh reads this file and generates .bki
@@ -1092,6 +1092,7 @@ DATA(insert OID = 782 (  gistbuild		   PGNSP PGUID 12 f f t f v 3 2278 "2281 228
 DESCR("gist(internal)");
 DATA(insert OID = 776 (  gistbulkdelete    PGNSP PGUID 12 f f t f v 3 2281 "2281 2281 2281" _null_ _null_ _null_ gistbulkdelete - _null_ ));
 DESCR("gist(internal)");
+DATA(insert OID = 2561 (  gistvacuumcleanup   PGNSP PGUID 12 f f t f v 3 2281 "2281 2281 2281" _null_ _null_ _null_ gistvacuumcleanup - _null_ ));
 DATA(insert OID = 772 (  gistcostestimate  PGNSP PGUID 12 f f t f v 7 2278 "2281 2281 2281 2281 2281 2281 2281" _null_ _null_ _null_  gistcostestimate - _null_ ));
 DESCR("gist(internal)");