diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index d4f4fd70c5c15bbe83fade9b7cdf094fbc96db33..25016c53375730f3264e0cd92717badc816dcc3e 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -168,7 +168,6 @@ typedef struct { uint32_t offset; uint64_t size; uint64_t tombSize; - uint32_t fver; } SDFInfo; typedef struct { @@ -176,6 +175,7 @@ typedef struct { TFILE f; int fd; uint8_t state; + uint32_t fver; } SDFile; void tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype); diff --git a/src/tsdb/inc/tsdbReadImpl.h b/src/tsdb/inc/tsdbReadImpl.h index 3df1f7a52a403032201c7400b47b80573c322255..613f6490fe3011169f25af207f10dad9985275a3 100644 --- a/src/tsdb/inc/tsdbReadImpl.h +++ b/src/tsdb/inc/tsdbReadImpl.h @@ -42,16 +42,16 @@ typedef struct { int32_t numOfRows : 24; int32_t len; int32_t keyLen; // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols + int16_t numOfSubBlocks; + int16_t numOfCols; // not including timestamp column + TSKEY keyFirst; + TSKEY keyLast; #ifdef __TD_6117__ int64_t hasAggr : 1; int64_t blkVer : 7; int64_t aggrOffset : 56; int32_t aggrLen; #endif - int16_t numOfSubBlocks; - int16_t numOfCols; // not including timestamp column - TSKEY keyFirst; - TSKEY keyLast; } SBlock; typedef struct { diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index ee507f03effa435c261ee26c958f53f313a61c03..b99dc9213760c88bb9c278885da5d68a4195dbcb 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -1166,6 +1166,7 @@ static int tsdbFetchTFileSet(STsdbRepo *pRepo, SArray **fArray) { return 0; } +#if 0 static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { const TFILE *pf = NULL; SArray * fArray = NULL; @@ -1188,7 +1189,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { TSDB_FSET_SET_INIT(&fset); // Loop to recover ONE fset - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX - 1; ftype++) { SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); if (index >= taosArrayGetSize(fArray)) { @@ -1268,6 +1269,7 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { } tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); + fset.nFiles = 3; taosArrayPush(pfs->cstatus->df, &fset); } @@ -1276,6 +1278,139 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { return 0; } +#endif + +#if 1 +static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { + const TFILE *pf = NULL; + SArray * fArray = NULL; + STsdbFS * pfs = REPO_FS(pRepo); + char dataDir[TSDB_FILENAME_LEN] = "\0"; + size_t fArraySize = 0; + + tsdbGetDataDir(REPO_ID(pRepo), dataDir); + + if (tsdbFetchTFileSet(pRepo, &fArray) < 0) { + tsdbError("vgId:%d failed to fetch TFileSet from %s to restore since %s", REPO_ID(pRepo), dataDir, + tstrerror(terrno)); + return -1; + } + + if ((fArraySize = taosArrayGetSize(fArray)) <= 0) { + tsdbInfo("vgId:%d size of DFileSet from %s is %" PRIu32, REPO_ID(pRepo), dataDir, (uint32_t)fArraySize); + return 0; + } + + // Loop to recover each file set + SDFileSet fset = {0}; + bool isOneFSetFinish = false; + // one fileset ends when (1) the array ends or (2) encounter different fid + for (size_t index = 0; index < fArraySize; ++index) { + int tvid = -1, tfid = -1; + TSDB_FILE_T ttype = TSDB_FILE_MAX; + uint32_t tversion = -1; + char bname[TSDB_FILENAME_LEN] = "\0"; + + pf = taosArrayGet(fArray, index); + tfsbasename(pf, bname); + tsdbParseDFilename(bname, &tvid, &tfid, &ttype, &tversion); + ASSERT(tvid == REPO_ID(pRepo)); + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ttype); + + if (tfid < pRepo->rtn.minFid) { // skip the file expired + continue; + } + + if (index == 0) { + memset(&fset, 0, sizeof(SDFileSet)); + TSDB_FSET_SET_INIT(&fset); + fset.nFiles = 1; + fset.fid = tfid; + pDFile->f = *pf; + isOneFSetFinish = false; + } else { + if (fset.fid == tfid) { + ++fset.nFiles; + pDFile->f = *pf; + // (1) the array ends + if ((index == fArraySize - 1) && (fset.nFiles >= TSDB_FILE_MIN)) { + tsdbInfo("vgId:%d DFileSet %d is fetched, nFiles=%" PRIu8, REPO_ID(pRepo), fset.fid, fset.nFiles); + isOneFSetFinish = true; + } + } else { + // (2) encounter different fid + if (fset.nFiles >= TSDB_FILE_MIN) { + tsdbInfo("vgId:%d DFileSet %d is fetched, nFiles=%" PRIu8, REPO_ID(pRepo), fset.fid, fset.nFiles); + isOneFSetFinish = true; + } else { + // next FSet + memset(&fset, 0, sizeof(SDFileSet)); + TSDB_FSET_SET_INIT(&fset); + fset.nFiles = 1; + fset.fid = tfid; + pDFile->f = *pf; + isOneFSetFinish = false; + continue; + } + } + } + + if (isOneFSetFinish) { + for (TSDB_FILE_T ftype = 0; ftype < fset.nFiles; ++ftype) { + SDFile * pDFile1 = TSDB_DFILE_IN_SET(&fset, ftype); + if (tsdbOpenDFile(pDFile1, O_RDONLY) < 0) { + tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile1), + tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + if (tsdbLoadDFileHeader(pDFile1, &(pDFile1->info)) < 0) { + tsdbError("vgId:%d failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile1), + tstrerror(terrno)); + taosArrayDestroy(fArray); + return -1; + } + + if (tsdbForceKeepFile) { + struct stat tfstat; + + // Get real file size + if (fstat(pDFile1->fd, &tfstat) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosArrayDestroy(fArray); + return -1; + } + + if (pDFile1->info.size != tfstat.st_size) { + int64_t tfsize = pDFile1->info.size; + pDFile1->info.size = tfstat.st_size; + tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile1), tfsize, pDFile1->info.size); + } + } + + tsdbCloseDFile(pDFile1); + } + tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid); + taosArrayPush(pfs->cstatus->df, &fset); + + // next FSet + memset(&fset, 0, sizeof(SDFileSet)); + TSDB_FSET_SET_INIT(&fset); + fset.nFiles = 1; + fset.fid = tfid; + pDFile->f = *pf; + isOneFSetFinish = false; + } + } + + // Resource release + taosArrayDestroy(fArray); + + return 0; +} +#endif static int tsdbRestoreCurrent(STsdbRepo *pRepo) { // Loop to recover mfile diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4e52ef7ac4ab22c8d5a190e0edb4e33c1ea1183e..89ab094395e24bef62da71d5bf47bba4970625d5 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -302,7 +302,6 @@ void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, memset(&(pDFile->info), 0, sizeof(pDFile->info)); pDFile->info.magic = TSDB_FILE_INIT_MAGIC; - pDFile->info.fver = tsdbGetDFSVersion(ftype); tsdbGetFilename(vid, fid, ver, ftype, fname); tfsInitFile(&(pDFile->f), did.level, did.id, fname); @@ -468,7 +467,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } void *pBuf = buf; - // pBuf = taosDecodeFixedU32(pBuf, &_version); + pBuf = taosDecodeFixedU32(pBuf, &(pDFile->fver)); pBuf = tsdbDecodeDFInfo(pBuf, pInfo); return 0; } @@ -528,7 +527,6 @@ static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) { static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { int tlen = 0; - tlen += taosEncodeFixedU32(buf, pInfo->fver); tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); @@ -541,7 +539,6 @@ static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) { } static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->fver)); buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index e991bf02aa68c92d7cf4dfdb09982ebaa6541bdc..75298a9e765edaf54681db52781fdc8316f7dafd 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -411,6 +411,7 @@ static int walSMemRowCheck(SWalHead *pHead) { pWalHead->len = pWalHead->len + lenExpand; } + ASSERT((sizeof(SWalHead) + pWalHead->len) <= WAL_MAX_SIZE); memcpy(pHead, pWalHead, sizeof(SWalHead) + pWalHead->len); tfree(pWalHead); }