diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6119f7086c6575a629dbb822425a775f394f9a3a..f42c00f1711a276c2a4d6682f93d0b92864aabb9 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -151,22 +151,22 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { -#ifdef TSDB_IDX - TSDB_FILE_TYPE_IDX = 0, - TSDB_FILE_TYPE_HEAD, -#else TSDB_FILE_TYPE_HEAD = 0, -#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_MAX, -#ifdef TSDB_IDX - TSDB_FILE_TYPE_NIDX, -#endif + TSDB_FILE_TYPE_STAT, TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NLAST + TSDB_FILE_TYPE_NDATA, + TSDB_FILE_TYPE_NLAST, + TSDB_FILE_TYPE_NSTAT } TSDB_FILE_TYPE; +#ifndef TDINTERNAL +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1) +#else +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1) +#endif + typedef struct { uint32_t magic; uint32_t len; @@ -281,9 +281,6 @@ typedef struct { TSKEY minKey; TSKEY maxKey; SFileGroup fGroup; -#ifdef TSDB_IDX - SFile nIdxF; -#endif SFile nHeadF; SFile nLastF; } SHelperFile; @@ -497,10 +494,6 @@ void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) -#ifdef TSDB_IDX -#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) -#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) @@ -512,7 +505,7 @@ int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); void tsdbDestroyHelper(SRWHelper* pHelper); void tsdbResetHelper(SRWHelper* pHelper); int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); -int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); +int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup); int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ee3bad9835bd5cb0be83d3a56102ac67771c5d4b..b8173d41b398747c4742a7c6e3cdd06b71030f67 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -21,11 +21,7 @@ #define TAOS_RANDOM_FILE_FAIL_TEST -#ifdef TSDB_IDX -const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; -#else -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; -#endif +const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); @@ -413,6 +409,10 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { pBuf = taosDecodeFixedU32(pBuf, &version); pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { + pFile->info.size = lseek(pFile->fd, 0, SEEK_END); + } + if (version != TSDB_FILE_VERSION) { tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 02c3ee4d85663ebbe5cc2300d993b98938d676ef..cd5f223f3da5c7595e3c267590b0be3047181daf 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -679,15 +679,10 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 0); + tsdbCloseHelperFile(pHelper, 0, pGroup); pthread_rwlock_wrlock(&(pFileH->fhlock)); -#ifdef TSDB_IDX - rename(helperNewIdxF(pHelper)->fname, helperIdxF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_IDX].info = helperNewIdxF(pHelper)->info; -#endif - rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; @@ -706,7 +701,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe _err: taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 1); + tsdbCloseHelperFile(pHelper, 1, NULL); return -1; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index d63bf8ab787646953f247aa8e021cb61659f1ad5..1e082e39bed065d6ae81a8aef53342a2f0535938 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -91,7 +91,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); // Reset the file part - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); pHelper->state = TSDB_HELPER_CLEAR_STATE; @@ -110,31 +110,16 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); -#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files -#ifdef TSDB_IDX - if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) return -1; -#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; -#ifdef TSDB_IDX - // Create and open .i file - pFile = helperNewIdxF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; -#endif - // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; @@ -161,14 +146,9 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return 0; } -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { +int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) { SFile *pFile = NULL; -#ifdef TSDB_IDX - pFile = helperIdxF(pHelper); - tsdbCloseFile(pFile); -#endif - pFile = helperHeadF(pHelper); tsdbCloseFile(pFile); @@ -177,10 +157,11 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } @@ -190,27 +171,16 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } -#endif - pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { if (!hasError) { @@ -412,10 +382,6 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { return -1; } -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); -#endif - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { @@ -426,6 +392,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + + pFile->info.size += pIdx->len; + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); } return 0; @@ -435,11 +404,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); off_t offset = 0; -#ifdef TSDB_IDX - SFile *pFile = helperNewIdxF(pHelper); -#else SFile *pFile = helperNewHeadF(pHelper); -#endif pFile->info.len += sizeof(TSCKSUM); if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { @@ -460,7 +425,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } - pFile->info.offset = offset; + ASSERT(offset == pFile->info.size); if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, @@ -469,16 +434,16 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } + pFile->info.offset = offset; + pFile->info.size += pFile->info.len; + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); -#ifdef TSDB_IDX - SFile *pFile = helperIdxF(pHelper); -#else SFile *pFile = helperHeadF(pHelper); -#endif int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { @@ -847,6 +812,9 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, pCompBlock->keyLast); + pFile->info.size += pCompBlock->len; + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; _err: @@ -1052,10 +1020,6 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { helperLastF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; -#ifdef TSDB_IDX - helperIdxF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; -#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { @@ -1064,7 +1028,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { } static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); taosTZfree(pHelper->idxH.pIdxArray); taosTZfree(pHelper->pWIdx);