diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 9346230e0294d26c8702586f32e73112d99f1be0..8ab89d72ed6db85d8ea14cc5c5884bdbf0690de0 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -132,12 +132,18 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { +#ifdef TSDB_IDX TSDB_FILE_TYPE_IDX = 0, TSDB_FILE_TYPE_HEAD, +#else + TSDB_FILE_TYPE_HEAD = 0, +#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, +#ifdef TSDB_IDX TSDB_FILE_TYPE_NIDX, +#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; @@ -147,6 +153,7 @@ typedef struct { uint32_t len; uint32_t totalBlocks; uint32_t totalSubBlocks; + uint32_t offset; uint64_t size; // total size of the file uint64_t tombSize; // unused file size } STsdbFileInfo; @@ -450,11 +457,13 @@ void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) +#ifdef TSDB_IDX #define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) +#define helperNewIdxF(h) (&((h)->files.nIdxF)) +#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) #define helperNewHeadF(h) (&((h)->files.nHeadF)) #define helperNewLastF(h) (&((h)->files.nLastF)) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 7b7788c4d9d70078d8fc24c93b089c79b97179b1..0a18d157737a2f2c191425959aabb6bd5db83e6a 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,7 +30,11 @@ #include "ttime.h" #include "tfile.h" +#ifdef TSDB_IDX const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; +#else +const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -108,7 +112,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); goto _err; @@ -126,7 +130,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); if (dir != NULL) closedir(dir); @@ -139,7 +143,7 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { tsdbDestroyFile(&pFGroup->files[type]); } } @@ -156,7 +160,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } @@ -169,7 +173,7 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int return pGroup; _err: - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } @@ -325,10 +329,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { int tlen = 0; tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->len); - tlen += taosEncodeFixedU64(buf, pInfo->size); - tlen += taosEncodeFixedU64(buf, pInfo->tombSize); tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); return tlen; } @@ -336,10 +341,11 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); return buf; } @@ -358,7 +364,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { pFileH->nFGroups--; ASSERT(pFileH->nFGroups >= 0); - for (int type = TSDB_FILE_TYPE_IDX; type < TSDB_FILE_TYPE_MAX; type++) { + for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { if (remove(fileGroup.files[type].fname) < 0) { tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 69e6ab3c0fa06d3b5beaac538ad7334cea557cbc..03fa9541c92925ced7d7f202c497771d53277ace 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -628,7 +628,9 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); +#ifdef TSDB_IDX pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); +#endif pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 19cb791ce897071b898a7a98aede62b1d157e96d..22182694deae35dc0db3c93dc9c8b7f00191583e 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -109,21 +109,27 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); +#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files +#ifdef TSDB_IDX if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; +#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; +#ifdef TSDB_IDX // Create and open .i file if (tsdbOpenFile(helperNewIdxF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; if (tsdbUpdateFileHeader(helperNewIdxF(pHelper), 0) < 0) return -1; +#endif // Create and open .h if (tsdbOpenFile(helperNewHeadF(pHelper), O_WRONLY | O_CREAT) < 0) return -1; @@ -150,11 +156,13 @@ _err: int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { SFile *pFile = NULL; +#ifdef TSDB_IDX pFile = helperIdxF(pHelper); if (pFile->fd > 0) { close(pFile->fd); pFile->fd = -1; } +#endif pFile = helperHeadF(pHelper); if (pFile->fd > 0) { @@ -182,6 +190,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (helperType(pHelper) == TSDB_WRITE_HELPER) { +#ifdef TSDB_IDX pFile = helperNewIdxF(pHelper); if (pFile->fd > 0) { if (!hasError) tsdbUpdateFileHeader(pFile, 0); @@ -200,6 +209,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { helperIdxF(pHelper)->info = pFile->info; } } +#endif pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { @@ -365,12 +375,13 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { int tsdbWriteCompInfo(SRWHelper *pHelper) { SCompIdx *pIdx = &(pHelper->curCompIdx); off_t offset = 0; + SFile * pFile = helperNewHeadF(pHelper); if (pIdx->len > 0) { if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) { - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseed file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -381,9 +392,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (tsendfile(helperNewHeadF(pHelper)->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { + if (tsendfile(pFile->fd, helperHeadF(pHelper)->fd, NULL, pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to send %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperHeadF(pHelper)->fname, helperNewHeadF(pHelper)->fname, strerror(errno)); + helperHeadF(pHelper)->fname, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -394,9 +405,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) && (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0); taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len); - offset = lseek(helperNewHeadF(pHelper)->fd, 0, SEEK_END); + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { - tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewHeadF(pHelper)->fname, + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -406,15 +417,19 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (twrite(helperNewHeadF(pHelper)->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, - helperNewHeadF(pHelper)->fname, strerror(errno)); + pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (tsizeof(pHelper->pWIdx) < helperNewIdxF(pHelper)->info.len + sizeof(SCompIdx) + 12) { +#ifdef TSDB_IDX + pFile = helperNewIdxF(pHelper); +#endif + + if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -422,8 +437,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } } - void *pBuf = POINTER_SHIFT(pHelper->pWIdx, helperNewIdxF(pHelper)->info.len); - helperNewIdxF(pHelper)->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); + pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); } return 0; @@ -431,9 +446,13 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); - // STsdbCfg *pCfg = &pHelper->pRepo->config; + off_t offset = 0; +#ifdef TSDB_IDX SFile *pFile = helperNewIdxF(pHelper); +#else + SFile *pFile = helperNewHeadF(pHelper); +#endif pFile->info.len += sizeof(TSCKSUM); if (tsizeof(pHelper->pWIdx) < pFile->info.len) { @@ -445,6 +464,15 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len); + offset = lseek(pFile->fd, 0, SEEK_END); + if (offset < 0) { + tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + pFile->info.offset = offset; + if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); @@ -457,8 +485,12 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); +#ifdef TSDB_IDX SFile *pFile = helperIdxF(pHelper); - int fd = pFile->fd; +#else + SFile *pFile = helperHeadF(pHelper); +#endif + int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object @@ -468,7 +500,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -516,11 +548,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - - // if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { - // terrno = TAOS_SYSTEM_ERROR(errno); - // return -1; - // } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -1031,13 +1058,15 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - helperIdxF(pHelper)->fd = -1; helperHeadF(pHelper)->fd = -1; helperDataF(pHelper)->fd = -1; helperLastF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; +#ifdef TSDB_IDX + helperIdxF(pHelper)->fd = -1; + helperNewIdxF(pHelper)->fd = -1; +#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) {