--force-compact-file

src/common/inc/tglobal.h

@@ -110,7 +110,8 @@ extern int8_t  tsCacheLastRow;
 
 //tsdb 
 extern bool tsdbForceKeepFile;
-extern bool tsdbEnableUpgradeFS;
+extern bool tsdbForceCompactFile;
+extern bool tsdbEnableUpgradeFile;
 
 #ifdef __TD_6117__
 extern bool tsdbQueryFromSMA;

src/common/src/tglobal.c

@@ -156,7 +156,8 @@ int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO;
 // tsdb config 
 // For backward compatibility
 bool tsdbForceKeepFile = false;
-bool tsdbEnableUpgradeFS = false;
+bool tsdbForceCompactFile = false; // compact TSDB fileset forcibly
+bool tsdbEnableUpgradeFile = false; // upgrade TSDB .head from v0 to v1+
 
 #ifdef __TD_6117__
 bool tsdbQueryFromSMA = true;

src/dnode/src/dnodeSystem.c

@@ -42,10 +42,12 @@ int32_t main(int32_t argc, char *argv[]) {
       }
     } else if (strcmp(argv[i], "-C") == 0) {
       dump_config = 1;
-    } else if (strcmp(argv[i], "--enable-upgrade-fs") == 0) {
-      tsdbEnableUpgradeFS = true;
+    } else if (strcmp(argv[i], "--enable-upgrade-file") == 0) {
+      tsdbEnableUpgradeFile = true;
     } else if (strcmp(argv[i], "--disable-query-from-sma") == 0) {
       tsdbQueryFromSMA = false;
+    } else if (strcmp(argv[i], "--force-compact-file") == 0) {
+      tsdbForceCompactFile = true;
     } else if (strcmp(argv[i], "--force-keep-file") == 0) {
       tsdbForceKeepFile = true;
     } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) {

src/tsdb/inc/tsdbReadImpl.h

@@ -236,7 +236,7 @@ int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
 void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
 int   tsdbLoadBlockIdx(SReadH *pReadh);
 int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
-int   tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen);
 int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
 int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
 int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);

src/tsdb/src/tsdbCompact.c

@@ -220,6 +220,9 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
   }
 
   static bool tsdbShouldCompact(SCompactH *pComph) {
+    if (tsdbForceCompactFile) {
+      return true;
+    }
     STsdbRepo *     pRepo = TSDB_COMPACT_REPO(pComph);
     STsdbCfg *      pCfg = REPO_CFG(pRepo);
     SReadH *        pReadh = &(pComph->readh);
@@ -390,7 +393,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) {
       // if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) {
       //   return -1;
       // }
-      int32_t originLen = 0;
+      uint32_t originLen = 0;
       if (tsdbLoadBlockInfo(pReadH, (void **)(&(pTh->pInfo)), &originLen) < 0) {
         return -1;
       }

src/tsdb/src/tsdbFS.c

@@ -341,7 +341,7 @@ int tsdbOpenFS(STsdbRepo *pRepo) {
     return -1;
   }
 
-  if (tsdbEnableUpgradeFS && tsdbRefactorFS(pRepo) < 0) {
+  if (tsdbEnableUpgradeFile && tsdbRefactorFS(pRepo) < 0) {
     tsdbError("vgId:%d failed to refactor FS since %s", REPO_ID(pRepo), tstrerror(terrno));
     return -1;
   }

src/tsdb/src/tsdbRead.c

@@ -1102,7 +1102,8 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int
   //   pCheckInfo->compSize = compIndex->len;
   // }
 
-  if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), &pCheckInfo->compSize) < 0) {
+  if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), (uint32_t*)(&pCheckInfo->compSize)) <
+      0) {
     return terrno;
   }
   SBlockInfo* pCompInfo = pCheckInfo->pCompInfo;

src/tsdb/src/tsdbReadImpl.c

@@ -269,16 +269,16 @@ static FORCE_INLINE size_t tsdbSizeOfSBlock(int32_t sBlkVer) {
   }
 }
 
-static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t srcBlkInfoLen, int32_t *dstBlkInfoLen) {
+static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t srcBlkInfoLen, uint32_t *dstBlkInfoLen) {
   int sBlkVer = tsdbGetSBlockVer(pHeadf->info.fver);
   if (sBlkVer == SBlockVerLatest) {
     *dstBlkInfoLen = srcBlkInfoLen;
     return TSDB_CODE_SUCCESS;
   }
-  uint32_t originBlkSize = tsdbSizeOfSBlock(sBlkVer);
-  int      nBlks = (srcBlkInfoLen - sizeof(SBlockInfo)) / originBlkSize;
+  size_t originBlkSize = tsdbSizeOfSBlock(sBlkVer);
+  size_t nBlks = (srcBlkInfoLen - sizeof(SBlockInfo)) / originBlkSize;
 
-  *dstBlkInfoLen = sizeof(SBlockInfo) + nBlks * sizeof(SBlock);
+  *dstBlkInfoLen = (uint32_t)(sizeof(SBlockInfo) + nBlks * sizeof(SBlock));
 
   if (srcBlkInfoLen == *dstBlkInfoLen) {
     return TSDB_CODE_SUCCESS;
@@ -300,7 +300,7 @@ static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t s
   return TSDB_CODE_SUCCESS;
 }
 
-int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen) {
+int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen) {
   ASSERT(pReadh->pBlkIdx != NULL);
 
   SDFile *    pHeadf = TSDB_READ_HEAD_FILE(pReadh);
@@ -337,7 +337,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen) {
 
   ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
 
-  int32_t dstBlkInfoLen = 0;
+  uint32_t dstBlkInfoLen = 0;
   if (tsdbHeadRefactor(pHeadf, &(pReadh->pBlkInfo), pBlkIdx->len, &dstBlkInfoLen) < 0) {
     return -1;
   }
@@ -422,7 +422,7 @@ static int tsdbLoadBlockStatisFromDFile(SReadH *pReadh, SBlock *pBlock) {
     return -1;
   }
 
-  size_t size = tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer);
+  size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
   if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1;
 
   int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
@@ -462,7 +462,7 @@ static int tsdbLoadBlockStatisFromAggr(SReadH *pReadh, SBlock *pBlock) {
     return -1;
   }
 
-  size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, pBlock->blkVer);
+  size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
   if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1;
 
   int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr);
@@ -647,7 +647,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
     return -1;
   }
 
-  int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer);
+  int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer);
   if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
     tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
@@ -846,8 +846,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
 
-  int64_t offset =
-      pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer) + tsdbGetBlockColOffset(pBlockCol);
+  int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) +
+                   tsdbGetBlockColOffset(pBlockCol);
   if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
     tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
               TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));

src/tsdb/src/tsdbRecover.c

@@ -226,10 +226,9 @@ static int tsdbDestroyHFile(SDFile *pDFile) {
 static int tsdbRefactorHeadF(STsdbRepo *pRepo, SRecoverH *pRecoverH, SDFileSet *pSet, int32_t *nRemain) {
   SDFile *pHeadF = TSDB_DFILE_IN_SET(pSet, TSDB_FILE_HEAD);
   if (pHeadF->info.fver == tsdbGetDFSVersion(TSDB_FILE_HEAD)) {
-    if (taosArrayPush(REPO_FS(pRepo)->nstatus->df, pSet) == NULL) {
-      terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    if (tsdbUpdateDFileSet(REPO_FS(pRepo),pSet) < 0) {
       return -1;
-    }
+    }    
     ++*nRemain;
     return TSDB_CODE_SUCCESS;
   }
@@ -321,8 +320,7 @@ static int tsdbRefactorHeadF(STsdbRepo *pRepo, SRecoverH *pRecoverH, SDFileSet *
   SDFileSet *pDestFSet = TSDB_READ_FSET(pReadH);
   tsdbInitDFileEx(TSDB_DFILE_IN_SET(pDestFSet, TSDB_FILE_HEAD), pTmpHeadF);
 
-  if (taosArrayPush(REPO_FS(pRepo)->nstatus->df, pDestFSet) == NULL) {
-    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+  if (tsdbUpdateDFileSet(REPO_FS(pRepo),pDestFSet) < 0) {
     return -1;
   }