diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8172ed4075dd2deff7b468fd8de624b9d0f44996..5cc17088f1dab7cd35228f21ef649a04911d83cb 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -219,6 +219,15 @@ int32_t tsdbLoadSBlockStatis(SDFileSetReader *pReader, SBlock *pBlock, SBlockSta // SDelFReader // tsdbUtil.c ============================================================================================== +int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size); +void tsdbFree(uint8_t *pBuf); + +// STMap +typedef struct STMap STMap; + +int32_t tPutTMap(uint8_t *p, STMap *pMap); +int32_t tGetTMap(uint8_t *p, STMap *pMap); + int32_t tTABLEIDCmprFn(const void *p1, const void *p2); int32_t tsdbKeyCmprFn(const void *p1, const void *p2); @@ -244,8 +253,6 @@ struct STsdb { STsdbFS *fs; }; -#if 1 // ====================================== - struct STable { uint64_t suid; uint64_t uid; @@ -771,29 +778,23 @@ typedef struct { } SDelDataItem; struct SDelData { - uint32_t delimiter; - tb_uid_t suid; - tb_uid_t uid; - int8_t flags; - int64_t nItem; - uint8_t *pOffset; - uint32_t nData; - uint8_t *pData; + int64_t version; + TSKEY sKey; + TSKEY eKey; }; -typedef struct { +struct SDelIdx { tb_uid_t suid; tb_uid_t uid; TSKEY minKey; TSKEY maxKey; - int64_t maxVersion; int64_t minVersion; + int64_t maxVersion; int64_t offset; int64_t size; -} SDelIdxItem; +}; -struct SDelIdx { - uint32_t delimiter; +struct STMap { uint8_t flags; uint32_t nOffset; uint8_t *pOffset; @@ -801,7 +802,14 @@ struct SDelIdx { uint8_t *pData; }; -#endif +struct SDelFile { + TSKEY minKey; + TSKEY maxKey; + int64_t minVersion; + int64_t maxVersion; + int64_t size; + int64_t offset; +}; #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index c664a53d7091fc14c81f400486d2d81426ad2752..c2e8de6a80d7af2cc3c73b2c7dc7060b914192ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -42,6 +42,8 @@ struct SCommitter { SDelFWriter *pDelFWriter; SDelIdx oDelIdx; SDelIdx nDelIdx; + SDelData oDelData; + SDelData nDelData; /* commit cache */ }; @@ -248,6 +250,21 @@ static int32_t tsdbCommitDelImpl(SCommitter *pCommitter) { static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; + code = tsdbWriteDelIdx(pCommitter->pDelFWriter, &pCommitter->nDelIdx, &pCommitter->pBuf3); + if (code) { + goto _err; + } + + code = tsdbDelFWriterClose(pCommitter->pDelFWriter); + if (code) { + goto _err; + } + + if (pCommitter->pDelFReader) { + code = tsdbDelFReaderClose(pCommitter->pDelFReader); + if (code) goto _err; + } + return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 0a21c4c253df3b7d21401fa58151e979dcb777ba..db9c70a98a926217ee8b50cea6ae21f7f5be5291 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -18,14 +18,6 @@ static const char *tsdbFileSuffix[] = {".tombstone", ".cache", ".index", ".data", ".last", ".sma", ""}; // .tombstone -struct SDelFile { - TSKEY minKey; - TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; - int64_t size; - int64_t offset; -}; struct STsdbIndexFile { int64_t size; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d4adb08b4c2a5c24bb6089fcdfddede5233f5f0a..2bfd54a77366b4bf68cc71275c37c73d48fb0613 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -105,6 +105,7 @@ int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SDelIdx *pDelIdx, uint8_t **ppBuf) // SDelFReader ==================================================== struct SDelFReader { + STsdb *pTsdb; SDelFile *pFile; TdFilePtr pReadH; }; @@ -129,6 +130,38 @@ int32_t tsdbReadDelData(SDelFReader *pReader, SDelData *pDelData, uint8_t **ppBu int32_t tsdbReadDelIdx(SDelFReader *pReader, SDelIdx *pDelIdx, uint8_t **ppBuf) { int32_t code = 0; - // TODO + int64_t offset = pReader->pFile->offset; + int64_t size = pReader->pFile->size - offset; + + // seek + if (taosLSeekFile(pReader->pReadH, pReader->pFile->offset, SEEK_SET) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // read + if (taosReadFile(pReader->pReadH, *ppBuf, size) < size) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + // realloc buf + code = tsdbRealloc(ppBuf, size); + if (code) { + goto _err; + } + + // check + if (!taosCheckChecksumWhole(*ppBuf, size)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + + // decode + + return code; + +_err: + tsdbError("vgId:%d failed to read del idx since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 5d9924bfeafd9b9b112fda80d2e042daad8b7504..96a386146baf6d79771bd5c0fc4947fb64b990e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -15,6 +15,41 @@ #include "tsdb.h" +int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size) { + int32_t code = 0; + int64_t bsize = 0; + uint8_t *pBuf; + + if (*ppBuf) { + bsize = *(int64_t *)((*ppBuf) - sizeof(int64_t)); + } + + if (bsize >= size) goto _exit; + + if (bsize == 0) bsize = 128; + while (bsize < size) { + bsize *= 2; + } + + pBuf = taosMemoryRealloc(*ppBuf ? (*ppBuf) - sizeof(int64_t) : *ppBuf, bsize + sizeof(int64_t)); + if (pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + *(int64_t *)pBuf = bsize; + *ppBuf = pBuf + sizeof(int64_t); + +_exit: + return code; +} + +void tsdbFree(uint8_t *pBuf) { + if (pBuf) { + taosMemoryFree(pBuf - sizeof(int64_t)); + } +} + int32_t tTABLEIDCmprFn(const void *p1, const void *p2) { TABLEID *pId1 = (TABLEID *)p1; TABLEID *pId2 = (TABLEID *)p2; @@ -53,24 +88,22 @@ int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } -int32_t tPutSDelIdx(uint8_t *p, SDelIdx *pDelIdx) { +int32_t tPutTMap(uint8_t *p, STMap *pMap) { int32_t n = 0; - n += tPutU32(p ? p + n : p, pDelIdx->delimiter); - n += tPutU8(p ? p + n : p, pDelIdx->flags); - n += tPutBinary(p ? p + n : p, pDelIdx->pOffset, pDelIdx->nOffset); - n += tPutBinary(p ? p + n : p, pDelIdx->pData, pDelIdx->nData); + n += tPutU8(p ? p + n : p, pMap->flags); + n += tPutBinary(p ? p + n : p, pMap->pOffset, pMap->nOffset); + n += tPutBinary(p ? p + n : p, pMap->pData, pMap->nData); return n; } -int32_t tGetSDelIdx(uint8_t *p, SDelIdx *pDelIdx) { +int32_t tGetTMap(uint8_t *p, STMap *pMap) { int32_t n = 0; - n += tGetU32(p + n, &pDelIdx->delimiter); - n += tGetU8(p + n, &pDelIdx->flags); - n += tGetBinary(p + n, &pDelIdx->pOffset, &pDelIdx->nOffset); - n += tGetBinary(p + n, &pDelIdx->pData, &pDelIdx->nData); + n += tGetU8(p, &pMap->flags); + n += tGetBinary(p, &pMap->pOffset, &pMap->nOffset); + n += tGetBinary(p, &pMap->pData, &pMap->nData); return n; } \ No newline at end of file