enh: add config for persist wal at the beginning of vnodeCommit

include/common/tglobal.h

@@ -125,6 +125,9 @@ extern char tsSmlChildTableName[];
 extern char tsSmlTagName[];
 extern bool tsSmlDataFormat;
 
+// wal
+extern int64_t tsWalRecoverSizeLimit;
+
 // internal
 extern int32_t tsTransPullupInterval;
 extern int32_t tsMqRebalanceInterval;

include/libs/wal/wal.h

@@ -43,7 +43,6 @@ extern "C" {
 #define WAL_FILE_LEN         (WAL_PATH_LEN + 32)
 #define WAL_MAGIC            0xFAFBFCFDF4F3F2F1ULL
 #define WAL_SCAN_BUF_SIZE    (1024 * 1024 * 3)
-#define WAL_RECOV_SIZE_LIMIT (200 * WAL_SCAN_BUF_SIZE)
 
 typedef enum {
   TAOS_WAL_WRITE = 1,

source/common/src/tglobal.c

@@ -156,6 +156,9 @@ char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPR
 // udf
 bool tsStartUdfd = true;
 
+// wal
+int64_t tsWalRecoverSizeLimit = (600 * 1024 * 1024L);
+
 // internal
 int32_t tsTransPullupInterval = 2;
 int32_t tsMqRebalanceInterval = 2;

source/libs/wal/src/walMeta.c

@@ -18,6 +18,7 @@
 #include "taoserror.h"
 #include "tutil.h"
 #include "walInt.h"
+#include "tglobal.h"
 
 bool FORCE_INLINE walLogExist(SWal* pWal, int64_t ver) {
   return !walIsEmpty(pWal) && walGetFirstVer(pWal) <= ver && walGetLastVer(pWal) >= ver;
@@ -82,11 +83,10 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
     readSize = end - offset;
     capacity = readSize + sizeof(magic);
 
-    int64_t limit = WAL_RECOV_SIZE_LIMIT;
-    if (limit < readSize) {
+    if (tsWalRecoverSizeLimit < readSize) {
       wError("vgId:%d, possibly corrupted WAL range exceeds size limit (i.e. %" PRId64 " bytes). offset:%" PRId64
              ", end:%" PRId64 ", file:%s",
-             pWal->cfg.vgId, limit, offset, end, fnameStr);
+             pWal->cfg.vgId, tsWalRecoverSizeLimit, offset, end, fnameStr);
       terrno = TSDB_CODE_WAL_SIZE_LIMIT;
       goto _err;
     }

source/libs/wal/src/walWrite.c

@@ -16,6 +16,7 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tchecksum.h"
+#include "tglobal.h"
 #include "walInt.h"
 
 int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) {
@@ -252,7 +253,7 @@ static FORCE_INLINE int32_t walCheckAndRoll(SWal *pWal) {
     }
   }
 
-  if (walGetLastFileCachedSize(pWal) > WAL_RECOV_SIZE_LIMIT / 2) {
+  if (walGetLastFileCachedSize(pWal) > tsWalRecoverSizeLimit / 2) {
     if (walSaveMeta(pWal) < 0) {
       return -1;
     }