Merge pull request #17721 from taosdata/FIX/TD-19673-3.0

enh: check completeness of WAL log entries after repairing

include/util/taoserror.h

@@ -451,6 +451,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_WAL_OUT_OF_MEMORY             TAOS_DEF_ERROR_CODE(0, 0x1004)
 #define TSDB_CODE_WAL_LOG_NOT_EXIST             TAOS_DEF_ERROR_CODE(0, 0x1005)
 #define TSDB_CODE_WAL_CHKSUM_MISMATCH           TAOS_DEF_ERROR_CODE(0, 0x1006)
+#define TSDB_CODE_WAL_LOG_INCOMPLETE            TAOS_DEF_ERROR_CODE(0, 0x1007)
 
 // tfs
 #define TSDB_CODE_FS_INVLD_CFG                  TAOS_DEF_ERROR_CODE(0, 0x2201)

source/libs/wal/src/walMeta.c

@@ -288,6 +288,33 @@ void walAlignVersions(SWal* pWal) {
   pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer);
 }
 
+bool walLogEntriesComplete(const SWal* pWal) {
+  int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
+  bool    complete = true;
+  int32_t fileIdx = -1;
+  int64_t index = pWal->vers.firstVer;
+
+  while (++fileIdx < sz) {
+    SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
+    if (pFileInfo->firstVer != index) {
+      break;
+    }
+    index = pFileInfo->lastVer + ((fileIdx + 1 < sz) ? 1 : 0);
+  }
+  // empty is regarded as complete
+  if (sz != 0) {
+    complete = (index == pWal->vers.lastVer);
+  }
+
+  if (!complete) {
+    wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], aligned with snaphotVer:%" PRId64,
+           pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer);
+    terrno = TSDB_CODE_WAL_LOG_INCOMPLETE;
+  }
+
+  return complete;
+}
+
 int walCheckAndRepairMeta(SWal* pWal) {
   // load log files, get first/snapshot/last version info
   const char* logPattern = "^[0-9]+.log$";
@@ -401,6 +428,11 @@ int walCheckAndRepairMeta(SWal* pWal) {
   if (updateMeta) {
     (void)walSaveMeta(pWal);
   }
+
+  if (!walLogEntriesComplete(pWal)) {
+    return -1;
+  }
+
   return 0;
 }
 

source/libs/wal/src/walWrite.c

@@ -101,7 +101,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
   taosThreadMutexLock(&pWal->mutex);
   int64_t code;
   char    fnameStr[WAL_FILE_LEN];
-  if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
+  if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) {
     terrno = TSDB_CODE_WAL_INVALID_VER;
     taosThreadMutexUnlock(&pWal->mutex);
     return -1;

source/libs/wal/test/walMetaTest.cpp

@@ -274,17 +274,17 @@ TEST_F(WalCleanEnv, rollbackMultiFile) {
   code = walRollback(pWal, 9);
   ASSERT_EQ(code, 0);
   ASSERT_EQ(pWal->vers.lastVer, 8);
-  code = walRollback(pWal, 5);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 4);
-  code = walRollback(pWal, 3);
+  code = walRollback(pWal, 6);
   ASSERT_EQ(code, 0);
+  ASSERT_EQ(pWal->vers.lastVer, 5);
+  code = walRollback(pWal, 5);
+  ASSERT_EQ(code, -1);
 
-  ASSERT_EQ(pWal->vers.lastVer, 2);
+  ASSERT_EQ(pWal->vers.lastVer, 5);
 
-  code = walWrite(pWal, 3, 3, (void*)ranStr, ranStrLen);
+  code = walWrite(pWal, 6, 6, (void*)ranStr, ranStrLen);
   ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 3);
+  ASSERT_EQ(pWal->vers.lastVer, 6);
 
   code = walSaveMeta(pWal);
   ASSERT_EQ(code, 0);

source/util/src/terror.c

@@ -452,6 +452,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER,              "WAL use invalid versi
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY,            "WAL out of memory")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST,            "WAL log not exist")
 TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH,          "WAL checksum mismatch")
+TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE,           "WAL log incomplete")
 
 // tfs
 TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG,                 "tfs invalid mount config")