From 6bdb6aad29746f690da4f6672f636630ccd91088 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 15 Sep 2021 06:25:50 +0800 Subject: [PATCH] --force-compact-file --- src/common/inc/tglobal.h | 3 ++- src/common/src/tglobal.c | 3 ++- src/dnode/src/dnodeSystem.c | 6 ++++-- src/tsdb/inc/tsdbReadImpl.h | 2 +- src/tsdb/src/tsdbCompact.c | 5 ++++- src/tsdb/src/tsdbFS.c | 2 +- src/tsdb/src/tsdbRead.c | 3 ++- src/tsdb/src/tsdbReadImpl.c | 22 +++++++++++----------- src/tsdb/src/tsdbRecover.c | 8 +++----- 9 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index b60c7e38dc..cc67ce31ed 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -110,7 +110,8 @@ extern int8_t tsCacheLastRow; //tsdb extern bool tsdbForceKeepFile; -extern bool tsdbEnableUpgradeFS; +extern bool tsdbForceCompactFile; +extern bool tsdbEnableUpgradeFile; #ifdef __TD_6117__ extern bool tsdbQueryFromSMA; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 020a15537b..3d679fca82 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -156,7 +156,8 @@ int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; -bool tsdbEnableUpgradeFS = false; +bool tsdbForceCompactFile = false; // compact TSDB fileset forcibly +bool tsdbEnableUpgradeFile = false; // upgrade TSDB .head from v0 to v1+ #ifdef __TD_6117__ bool tsdbQueryFromSMA = true; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 10ee371698..9295743795 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,10 +42,12 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; - } else if (strcmp(argv[i], "--enable-upgrade-fs") == 0) { - tsdbEnableUpgradeFS = true; + } else if (strcmp(argv[i], "--enable-upgrade-file") == 0) { + tsdbEnableUpgradeFile = true; } else if (strcmp(argv[i], "--disable-query-from-sma") == 0) { tsdbQueryFromSMA = false; + } else if (strcmp(argv[i], "--force-compact-file") == 0) { + tsdbForceCompactFile = true; } else if (strcmp(argv[i], "--force-keep-file") == 0) { tsdbForceKeepFile = true; } else if (strcmp(argv[i], "--compact-mnode-wal") == 0) { diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h index 38ad580a57..7cec7a671f 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -236,7 +236,7 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet); void tsdbCloseAndUnsetFSet(SReadH *pReadh); int tsdbLoadBlockIdx(SReadH *pReadh); int tsdbSetReadTable(SReadH *pReadh, STable *pTable); -int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen); +int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen); int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds); int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 025e29ae4e..4fdd293d70 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -220,6 +220,9 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { } static bool tsdbShouldCompact(SCompactH *pComph) { + if (tsdbForceCompactFile) { + return true; + } STsdbRepo * pRepo = TSDB_COMPACT_REPO(pComph); STsdbCfg * pCfg = REPO_CFG(pRepo); SReadH * pReadh = &(pComph->readh); @@ -390,7 +393,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { // if (tsdbMakeRoom((void **)(&(pTh->pInfo)), pTh->pBlkIdx->len) < 0) { // return -1; // } - int32_t originLen = 0; + uint32_t originLen = 0; if (tsdbLoadBlockInfo(pReadH, (void **)(&(pTh->pInfo)), &originLen) < 0) { return -1; } diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 2affbc5114..d7ab7a6fca 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -341,7 +341,7 @@ int tsdbOpenFS(STsdbRepo *pRepo) { return -1; } - if (tsdbEnableUpgradeFS && tsdbRefactorFS(pRepo) < 0) { + if (tsdbEnableUpgradeFile && tsdbRefactorFS(pRepo) < 0) { tsdbError("vgId:%d failed to refactor FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 07a47084e1..ce89c0c10b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1102,7 +1102,8 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int // pCheckInfo->compSize = compIndex->len; // } - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), &pCheckInfo->compSize) < 0) { + if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void**)(&pCheckInfo->pCompInfo), (uint32_t*)(&pCheckInfo->compSize)) < + 0) { return terrno; } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 0594fb4a39..f53afa5de3 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -269,16 +269,16 @@ static FORCE_INLINE size_t tsdbSizeOfSBlock(int32_t sBlkVer) { } } -static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t srcBlkInfoLen, int32_t *dstBlkInfoLen) { +static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t srcBlkInfoLen, uint32_t *dstBlkInfoLen) { int sBlkVer = tsdbGetSBlockVer(pHeadf->info.fver); if (sBlkVer == SBlockVerLatest) { *dstBlkInfoLen = srcBlkInfoLen; return TSDB_CODE_SUCCESS; } - uint32_t originBlkSize = tsdbSizeOfSBlock(sBlkVer); - int nBlks = (srcBlkInfoLen - sizeof(SBlockInfo)) / originBlkSize; + size_t originBlkSize = tsdbSizeOfSBlock(sBlkVer); + size_t nBlks = (srcBlkInfoLen - sizeof(SBlockInfo)) / originBlkSize; - *dstBlkInfoLen = sizeof(SBlockInfo) + nBlks * sizeof(SBlock); + *dstBlkInfoLen = (uint32_t)(sizeof(SBlockInfo) + nBlks * sizeof(SBlock)); if (srcBlkInfoLen == *dstBlkInfoLen) { return TSDB_CODE_SUCCESS; @@ -300,7 +300,7 @@ static int tsdbHeadRefactor(SDFile *pHeadf, SBlockInfo **pDstBlkInfo, uint32_t s return TSDB_CODE_SUCCESS; } -int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen) { +int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, uint32_t *extendedLen) { ASSERT(pReadh->pBlkIdx != NULL); SDFile * pHeadf = TSDB_READ_HEAD_FILE(pReadh); @@ -337,7 +337,7 @@ int tsdbLoadBlockInfo(SReadH *pReadh, void **pTarget, int32_t *extendedLen) { ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid); - int32_t dstBlkInfoLen = 0; + uint32_t dstBlkInfoLen = 0; if (tsdbHeadRefactor(pHeadf, &(pReadh->pBlkInfo), pBlkIdx->len, &dstBlkInfoLen) < 0) { return -1; } @@ -422,7 +422,7 @@ static int tsdbLoadBlockStatisFromDFile(SReadH *pReadh, SBlock *pBlock) { return -1; } - size_t size = tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer); + size_t size = tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1; int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); @@ -462,7 +462,7 @@ static int tsdbLoadBlockStatisFromAggr(SReadH *pReadh, SBlock *pBlock) { return -1; } - size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, pBlock->blkVer); + size_t sizeAggr = tsdbBlockAggrSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (tsdbMakeRoom((void **)(&(pReadh->pAggrBlkData)), sizeAggr) < 0) return -1; int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); @@ -647,7 +647,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return -1; } - int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer); + int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", @@ -846,8 +846,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; - int64_t offset = - pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, pBlock->blkVer) + tsdbGetBlockColOffset(pBlockCol); + int64_t offset = pBlock->offset + tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer) + + tsdbGetBlockColOffset(pBlockCol); if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno)); diff --git a/src/tsdb/src/tsdbRecover.c b/src/tsdb/src/tsdbRecover.c index 0e575a95a9..d0022b0135 100644 --- a/src/tsdb/src/tsdbRecover.c +++ b/src/tsdb/src/tsdbRecover.c @@ -226,10 +226,9 @@ static int tsdbDestroyHFile(SDFile *pDFile) { static int tsdbRefactorHeadF(STsdbRepo *pRepo, SRecoverH *pRecoverH, SDFileSet *pSet, int32_t *nRemain) { SDFile *pHeadF = TSDB_DFILE_IN_SET(pSet, TSDB_FILE_HEAD); if (pHeadF->info.fver == tsdbGetDFSVersion(TSDB_FILE_HEAD)) { - if (taosArrayPush(REPO_FS(pRepo)->nstatus->df, pSet) == NULL) { - terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + if (tsdbUpdateDFileSet(REPO_FS(pRepo),pSet) < 0) { return -1; - } + } ++*nRemain; return TSDB_CODE_SUCCESS; } @@ -321,8 +320,7 @@ static int tsdbRefactorHeadF(STsdbRepo *pRepo, SRecoverH *pRecoverH, SDFileSet * SDFileSet *pDestFSet = TSDB_READ_FSET(pReadH); tsdbInitDFileEx(TSDB_DFILE_IN_SET(pDestFSet, TSDB_FILE_HEAD), pTmpHeadF); - if (taosArrayPush(REPO_FS(pRepo)->nstatus->df, pDestFSet) == NULL) { - terrno = TSDB_CODE_FS_OUT_OF_MEMORY; + if (tsdbUpdateDFileSet(REPO_FS(pRepo),pDestFSet) < 0) { return -1; } -- GitLab