diff --git a/src/backend/access/bitmap/bitmap.c b/src/backend/access/bitmap/bitmap.c index a678bef0a8e8f4c6f938e959086d0ba5fcce6153..c9f36e875974ceb11a8c5d12bdcc689b6eb532f5 100644 --- a/src/backend/access/bitmap/bitmap.c +++ b/src/backend/access/bitmap/bitmap.c @@ -66,7 +66,6 @@ bmbuild(PG_FUNCTION_ARGS) BMBuildState bmstate; IndexBuildResult *result; TupleDesc tupDesc; - bool useWal; MIRROREDLOCK_BUFMGR_VERIFY_NO_LOCK_LEAK_ENTER; @@ -85,10 +84,8 @@ bmbuild(PG_FUNCTION_ARGS) tupDesc = RelationGetDescr(index); - useWal = (!XLog_UnconvertedCanBypassWal() && !index->rd_istemp); - /* initialize the bitmap index. */ - _bitmap_init(index, useWal); + _bitmap_init(index, !index->rd_istemp); /* initialize the build state. */ _bitmap_init_buildstate(index, &bmstate); @@ -98,23 +95,6 @@ bmbuild(PG_FUNCTION_ARGS) bmbuildCallback, (void *)&bmstate); /* clean up the build state */ _bitmap_cleanup_buildstate(index, &bmstate); - - /* - * fsync the relevant files to disk, unless we're building - * a temporary index - */ - if (!useWal) - { - FlushRelationBuffers(bmstate.bm_lov_heap); - smgrimmedsync(bmstate.bm_lov_heap->rd_smgr); - - FlushRelationBuffers(bmstate.bm_lov_index); - smgrimmedsync(bmstate.bm_lov_index->rd_smgr); - - FlushRelationBuffers(index); - /* FlushRelationBuffers will have opened rd_smgr */ - smgrimmedsync(index->rd_smgr); - } /* return statistics */ result = (IndexBuildResult *) palloc0(sizeof(IndexBuildResult)); diff --git a/src/backend/access/bitmap/bitmapattutil.c b/src/backend/access/bitmap/bitmapattutil.c index a243ac7208a768b9a47b42b4ec7ec9f861e22835..57be821df364b7e7053775caa690fdeff533fd34 100644 --- a/src/backend/access/bitmap/bitmapattutil.c +++ b/src/backend/access/bitmap/bitmapattutil.c @@ -117,7 +117,8 @@ _bitmap_create_lov_heapandindex(Relation rel, _bt_initmetapage(btree_metapage, P_NONE, 0); /* XLOG the metapage */ - if (!XLog_UnconvertedCanBypassWal() && !lovIndex->rd_istemp) + + if (!lovIndex->rd_istemp) { // Fetch gp_persistent_relation_node information that will be added to XLOG record. RelationFetchGpRelationNodeForXLog(lovIndex); diff --git a/src/backend/access/bitmap/bitmappages.c b/src/backend/access/bitmap/bitmappages.c index 18a9967f0846b8c158ea3e1f647dd9ae2d7ac9db..5b5d388c6dc2b41b2700c5e21a5456fda74828c8 100644 --- a/src/backend/access/bitmap/bitmappages.c +++ b/src/backend/access/bitmap/bitmappages.c @@ -306,7 +306,7 @@ _bitmap_init_buildstate(Relation index, BMBuildState *bmstate) * writes page to the shared buffer, we can't disable WAL archiving. * We will add this shortly. */ - bmstate->use_wal = !XLog_UnconvertedCanBypassWal() && !index->rd_istemp; + bmstate->use_wal = !index->rd_istemp; } /* diff --git a/src/backend/access/nbtree/nbtsort.c b/src/backend/access/nbtree/nbtsort.c index ebfb8d64ff094abebfaad4ae3c276e5a36371c3c..1c79c1540b78a156b61ba889243acf8f9114b075 100644 --- a/src/backend/access/nbtree/nbtsort.c +++ b/src/backend/access/nbtree/nbtsort.c @@ -209,10 +209,10 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2) wstate.index = btspool->index; /* - * We need to log index creation in WAL iff WAL archiving is enabled AND - * it's not a temp index. + * We need to log index creation in WAL iff WAL archiving/streaming is + * enabled AND it's not a temp index. */ - wstate.btws_use_wal = !XLog_UnconvertedCanBypassWal() && !wstate.index->rd_istemp; + wstate.btws_use_wal = XLogIsNeeded() && !wstate.index->rd_istemp; /* reserve the metapage */ wstate.btws_pages_alloced = BTREE_METAPAGE + 1; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 2b7a848cd5396a681a0bc891503755cf33b62cf2..bf2351de0df9c859d723126aa2404260a9b7cf05 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -720,40 +720,6 @@ static bool CheckForStandbyTrigger(void); static void GetXLogCleanUpTo(XLogRecPtr recptr, uint32 *_logId, uint32 *_logSeg); static void checkXLogConsistency(XLogRecord *record, XLogRecPtr EndRecPtr); -/* - * Whether we need to always generate transaction log (XLOG), or if we can - * bypass it and get better performance. - * - * For GPDB, we currently do not support XLogArchivingActive(), so we don't - * use it as a condition. - */ -bool XLog_CanBypassWal(void) -{ -#ifdef USE_SEGWALREP - /* - * Wal replication enabled for segments, shouldn't skip anything from - * wal. - */ - return false; -#else - /* - * We need the XLOG to be transmitted to the standby master since it is - * not using FileRep technology yet. Master also could skip some of the - * WAL operations for optimization when standby is not configured, but for - * now we lean towards safety. - */ - return GpIdentity.segindex != MASTER_CONTENT_ID; -#endif -} - -/* - * For FileRep code that doesn't have the Bypass WAL logic yet. - */ -bool XLog_UnconvertedCanBypassWal(void) -{ - return false; -} - static char *XLogContiguousCopy( XLogRecord *record, @@ -3071,15 +3037,16 @@ XLogFileClose(void) /* * WAL segment files will not be re-read in normal operation, so we advise - * OS to release any cached pages. But do not do so if WAL archiving is - * active, because archiver process could use the cache to read the WAL - * segment. + * OS to release any cached pages. But do not do so if WAL archiving or + * streaming is active, because archiver process could use the cache to + * read the WAL segment. Also, don't bother with it if we are using + * O_DIRECT, since the kernel is presumably not caching in that case. * * While O_DIRECT works for O_SYNC, posix_fadvise() works for fsync() and * O_SYNC, and some platforms only have posix_fadvise(). */ #if defined(HAVE_DECL_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED) - if (!XLogArchivingActive()) + if (!XLogIsNeeded()) posix_fadvise(openLogFile, 0, 0, POSIX_FADV_DONTNEED); #endif #endif /* NOT_USED */ diff --git a/src/backend/commands/cluster.c b/src/backend/commands/cluster.c index 8443c6512342abbe3c534b2774e88490ef3e96d1..60e74bf186249823163a5f3dcca30b3236e42b57 100644 --- a/src/backend/commands/cluster.c +++ b/src/backend/commands/cluster.c @@ -849,10 +849,10 @@ copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex) LockRelationOid(OldHeap->rd_rel->reltoastrelid, AccessExclusiveLock); /* - * We need to log the copied data in WAL iff WAL archiving is enabled AND - * it's not a temp rel. + * We need to log the copied data in WAL iff WAL archiving/streaming is + * enabled AND it's not a temp rel. */ - use_wal = XLogArchivingActive() && !NewHeap->rd_istemp; + use_wal = XLogIsNeeded() && !NewHeap->rd_istemp; /* use_wal off requires rd_targblock be initially invalid */ Assert(NewHeap->rd_targblock == InvalidBlockNumber); diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 8c46ebd0e01e946471a7341db7c28ad177b35dd0..4e2679123d4d3d2e3a01720357a926deb1aa3aad 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -4275,7 +4275,7 @@ CopyFrom(CopyState cstate) /*---------- * Check to see if we can avoid writing WAL * - * If archive logging is not enabled *and* either + * If archive logging/streaming is not enabled *and* either * - table was created in same transaction as this COPY * - data is being written to relfilenode created in this transaction * then we can skip writing WAL. It's safe because if the transaction @@ -4303,8 +4303,7 @@ CopyFrom(CopyState cstate) cstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId) { use_fsm = false; - if (!XLogArchivingActive()) - use_wal = false; + use_wal = XLogIsNeeded(); } oldcontext = MemoryContextSwitchTo(estate->es_query_cxt); diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 709dd9a83d5ba986c6e343c3e6f343aa93cd8826..09d87ee015124611ebf300995eaeb7d43e84fb95 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -1170,7 +1170,7 @@ createdb(CreatedbStmt *stmt) PersistentFileSysRelStorageMgr localRelStorageMgr; PersistentFileSysRelBufpoolKind relBufpoolKind; - useWal = !XLog_CanBypassWal(); + useWal = XLogIsNeeded(); GpPersistentRelationNode_GetRelationInfo( dbInfoRel->relkind, diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 151a5140396b7949f3575ee9eb7557c6782bcadb..7e0a072a261d592876fa5f9d7da77478bec41d41 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -10059,7 +10059,7 @@ ATExecSetTableSpace_BufferPool( * We need to log the copied data in WAL enabled AND * it's not a temp rel. */ - useWal = !XLog_CanBypassWal() && !rel->rd_istemp; + useWal = XLogIsNeeded() && !rel->rd_istemp; if (Debug_persistent_print) elog(Persistent_DebugPrintLevel(), diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c index ba53bdb6654a75f0d3835ee1cd54579de93d5df8..1a8356ebbb65a5449bdcd6fb290b93a3d72cc902 100644 --- a/src/backend/executor/execMain.c +++ b/src/backend/executor/execMain.c @@ -4789,7 +4789,7 @@ OpenIntoRel(QueryDesc *queryDesc) */ bufferPoolBulkLoad = (relstorage_is_buffer_pool(relstorage) ? - XLog_CanBypassWal() : false); + !XLogIsNeeded() : false); /* Now we can actually create the new relation */ intoRelationId = heap_create_with_catalog(intoName, diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 717c43d9df273fccda4da5da0f4cba8f4174eaa4..7e1e25ce936fbcac04b2e730a3f4b18a83216099 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -21,6 +21,7 @@ #include "utils/relcache.h" #include "utils/timestamp.h" #include "cdb/cdbpublic.h" +#include "replication/walsender.h" /* * REDO Tracking DEFINEs. @@ -187,18 +188,12 @@ extern bool log_checkpoints; #define XLogArchivingActive() (XLogArchiveMode) #define XLogArchiveCommandSet() (XLogArchiveCommand[0] != '\0') -/* - * Whether we need to always generate transaction log (XLOG), or if we can - * bypass it and get better performance. - * - * For GPDB, we do not support XLogArchivingActive(), so we don't use it as a condition. - */ -extern bool XLog_CanBypassWal(void); - /* - * For FileRep code that doesn't have the Bypass WAL logic yet. + * Is WAL-logging necessary? We need to log an XLOG record iff either + * WAL archiving is enabled or XLOG streaming is allowed. */ -extern bool XLog_UnconvertedCanBypassWal(void); + +#define XLogIsNeeded() (XLogArchivingActive() || (max_wal_senders > 0)) extern bool am_startup; diff --git a/src/test/regress/input/gpcopy.source b/src/test/regress/input/gpcopy.source index 65b8d90954c0188a67bc04ef4a285118863f785e..bc5006fe30573c4a39e475f0b29ccad7445ee40e 100644 --- a/src/test/regress/input/gpcopy.source +++ b/src/test/regress/input/gpcopy.source @@ -1099,3 +1099,15 @@ SET gp_enable_segment_copy_checking=off; COPY COPY_FROM_PROGRAM_ERROR FROM PROGRAM $quote$bash -c "echo -e '$GP_SEGMENT_ID\n'"$quote$ on segment; COPY COPY_FROM_PROGRAM_ERROR FROM PROGRAM $quote$bash -c "echo -e '$GP_SEGMENT_I\n'"$quote$ on segment; SET gp_enable_segment_copy_checking=on; + +-- Test that xlog records are generated for COPY in the same transaction as +-- created table. The actual validation for this test will be performed once the +-- gp_replica_check tool is enabled. +BEGIN; +CREATE TABLE copy_from_same_txn(a int, b int); +COPY copy_from_same_txn FROM stdin; +1 1 +2 2 +3 3 +\. +COMMIT; diff --git a/src/test/regress/output/gpcopy.source b/src/test/regress/output/gpcopy.source index 90f44295f595d44e5b64ea7cf91b6b3145b7d235..9bd0950fc8291fb773dc124eba1d85a2c5722bcb 100755 --- a/src/test/regress/output/gpcopy.source +++ b/src/test/regress/output/gpcopy.source @@ -1298,3 +1298,12 @@ COPY COPY_FROM_PROGRAM_ERROR FROM PROGRAM $quote$bash -c "echo -e '$GP_SEGMENT_I ERROR: invalid input syntax for integer: "" (seg0 172.17.0.2:40000 pid=27935) CONTEXT: COPY copy_from_program_error, line 1, column SET gp_enable_segment_copy_checking=on; +-- Test that xlog records are generated for COPY in the same transaction as +-- created table. The actual validation for this test will be performed once the +-- gp_replica_check tool is enabled. +BEGIN; +CREATE TABLE copy_from_same_txn(a int, b int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +COPY copy_from_same_txn FROM stdin; +COMMIT;