From 4e2c2b80842bd1ebdbdac3f150705e67f5f9018e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 11 Jun 2022 12:02:12 +0000 Subject: [PATCH] more work --- include/util/taoserror.h | 1 + source/dnode/vnode/src/inc/tsdb.h | 9 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 30 ++--- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 116 +++++++++++++++++- source/util/src/terror.c | 1 + 6 files changed, 135 insertions(+), 24 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03308e395f..40fb024719 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -71,6 +71,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0029) #define TSDB_CODE_INVALID_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x0030) #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) +#define TSDB_CODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0032) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b83638d0f4..cb6df89c07 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -22,6 +22,10 @@ extern "C" { #endif +#define TSDB_OFFSET_U8 ((uint8_t)0x1) +#define TSDB_OFFSET_U16 ((uint8_t)0x2) +#define TSDB_OFFSET_U32 ((uint8_t)0x4) + // tsdbDebug ================ // clang-format off #define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) @@ -150,7 +154,8 @@ int32_t tsdbKeyCmprFn(const void *p1, const void *p2); // SDelIdx int32_t tDelIdxGetSize(SDelIdx *pDelIdx); -int32_t tDelIdxGetItem(SDelIdx *pDelIdx, int32_t idx, SDelIdxItem *pItem); +int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id); +int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx); int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem); int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx); int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx); @@ -375,7 +380,7 @@ struct SDelIdxItem { struct SDelIdx { uint32_t delimiter; uint8_t flags; - uint32_t nOffset; + uint32_t nItem; uint8_t *pOffset; uint32_t nData; uint8_t *pData; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7349053017..e73763938c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -42,10 +42,10 @@ struct SCommitter { /* commit del */ SDelFReader *pDelFReader; SDelFWriter *pDelFWriter; - SDelIdx oDelIdx; - SDelIdx nDelIdx; - SDelData oDelData; - SDelData nDelData; + SDelIdx delIdxOld; + SDelIdx delIdxNew; + SDelData delDataOld; + SDelData delDataNew; SDelIdxItem delIdxItem; /* commit cache */ }; @@ -175,21 +175,21 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { SDelFile *pDelFileW = NULL; // TODO // load old - pCommitter->oDelIdx = (SDelIdx){0}; + pCommitter->delIdxOld = (SDelIdx){0}; if (pDelFileR) { code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb, NULL); if (code) { goto _err; } - code = tsdbReadDelIdx(pCommitter->pDelFReader, &pCommitter->oDelIdx, &pCommitter->pBuf1); + code = tsdbReadDelIdx(pCommitter->pDelFReader, &pCommitter->delIdxOld, &pCommitter->pBuf1); if (code) { goto _err; } } // prepare new - pCommitter->nDelIdx = (SDelIdx){0}; + pCommitter->delIdxNew = (SDelIdx){0}; code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, pDelFileW, pTsdb); if (code) { goto _err; @@ -245,7 +245,7 @@ _err: static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; - code = tsdbWriteDelIdx(pCommitter->pDelFWriter, &pCommitter->nDelIdx, NULL); + code = tsdbWriteDelIdx(pCommitter->pDelFWriter, &pCommitter->delIdxNew, NULL); if (code) { goto _err; } @@ -304,7 +304,7 @@ _exit: return code; _err: - tsdbError("vgId:%d commit del data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -543,14 +543,14 @@ static int32_t tsdbCommitTableDelStart(SCommitter *pCommitter) { int32_t code = 0; // load old - pCommitter->oDelData = (SDelData){0}; + pCommitter->delDataOld = (SDelData){0}; if (0) { - code = tsdbReadDelData(pCommitter->pDelFReader, NULL /*TODO*/, &pCommitter->oDelData, &pCommitter->pBuf4); + code = tsdbReadDelData(pCommitter->pDelFReader, NULL /*TODO*/, &pCommitter->delDataOld, &pCommitter->pBuf4); if (code) goto _err; } // prepare new - pCommitter->nDelData = (SDelData){0}; + pCommitter->delDataNew = (SDelData){0}; return code; @@ -566,7 +566,7 @@ static int32_t tsdbCommitTableDelImpl(SCommitter *pCommitter) { for (; pDelOp; pDelOp = pDelOp->pNext) { SDelDataItem item = {.version = pDelOp->version, .sKey = pDelOp->sKey, .eKey = pDelOp->eKey}; - code = tDelDataPutItem(&pCommitter->nDelData, &item); + code = tDelDataPutItem(&pCommitter->delDataNew, &item); if (code) goto _err; } @@ -580,11 +580,11 @@ static int32_t tsdbCommitTableDelEnd(SCommitter *pCommitter) { int32_t code = 0; // write table del data - code = tsdbWriteDelData(pCommitter->pDelFWriter, &pCommitter->nDelData, NULL); + code = tsdbWriteDelData(pCommitter->pDelFWriter, &pCommitter->delDataNew, NULL); if (code) goto _err; // add SDelIdxItem - code = tDelIdxPutItem(&pCommitter->nDelIdx, &pCommitter->delIdxItem); + code = tDelIdxPutItem(&pCommitter->delIdxNew, &pCommitter->delIdxItem); if (code) goto _err; return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 49406961aa..1bf5c9b011 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -410,7 +410,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SDelIdx *pDelIdx, uint8_t **ppBuf) int32_t n = tGetDelIdx(*ppBuf, pDelIdx); ASSERT(n == size - sizeof(TSCKSUM)); ASSERT(pDelIdx->delimiter == TSDB_FILE_DLMT); - ASSERT(pDelIdx->nOffset > 0 && pDelIdx->nData > 0); + // ASSERT(pDelIdx->nOffset > 0 && pDelIdx->nData > 0); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c1093583dd..b284232754 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -15,6 +15,30 @@ #include "tsdb.h" +static FORCE_INLINE int32_t tsdbOffsetSize(uint8_t flags) { + if (flags & TSDB_OFFSET_U8) { + return sizeof(uint8_t); + } else if (flags & TSDB_OFFSET_U16) { + return sizeof(uint16_t); + } else if (flags & TSDB_OFFSET_U32) { + return sizeof(uint32_t); + } else { + ASSERT(0); + } +} + +static FORCE_INLINE uint32_t tsdbGetOffset(uint8_t *pOffset, uint8_t flags, int32_t idx) { + if (flags & TSDB_OFFSET_U8) { + return ((uint8_t *)pOffset)[idx]; + } else if (flags & TSDB_OFFSET_U16) { + return ((uint16_t *)pOffset)[idx]; + } else if (flags & TSDB_OFFSET_U32) { + return ((uint32_t *)pOffset)[idx]; + } else { + ASSERT(0); + } +} + int32_t tsdbRealloc(uint8_t **ppBuf, int64_t size) { int32_t code = 0; int64_t bsize = 0; @@ -88,6 +112,37 @@ int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { return 0; } +// SDelIdxItem ====================================================== +static FORCE_INLINE int32_t tPutDelIdxItem(uint8_t *p, SDelIdxItem *pDelIdxItem) { + int32_t n = 0; + + n += tPutI64(p ? p + n : p, pDelIdxItem->suid); + n += tPutI64(p ? p + n : p, pDelIdxItem->uid); + n += tPutI64(p ? p + n : p, pDelIdxItem->minKey); + n += tPutI64(p ? p + n : p, pDelIdxItem->maxKey); + n += tPutI64v(p ? p + n : p, pDelIdxItem->minVersion); + n += tPutI64v(p ? p + n : p, pDelIdxItem->maxVersion); + n += tPutI64v(p ? p + n : p, pDelIdxItem->offset); + n += tPutI64v(p ? p + n : p, pDelIdxItem->size); + + return n; +} + +static FORCE_INLINE int32_t tGetDelIdxItem(uint8_t *p, SDelIdxItem *pDelIdxItem) { + int32_t n = 0; + + n += tGetI64(p, &pDelIdxItem->suid); + n += tGetI64(p, &pDelIdxItem->uid); + n += tGetI64(p, &pDelIdxItem->minKey); + n += tGetI64(p, &pDelIdxItem->maxKey); + n += tGetI64v(p, &pDelIdxItem->minVersion); + n += tGetI64v(p, &pDelIdxItem->maxVersion); + n += tGetI64v(p, &pDelIdxItem->offset); + n += tGetI64v(p, &pDelIdxItem->size); + + return n; +} + // SDelIdx ====================================================== int32_t tDelIdxGetSize(SDelIdx *pDelIdx) { int32_t code = 0; @@ -95,15 +150,62 @@ int32_t tDelIdxGetSize(SDelIdx *pDelIdx) { return code; } -int32_t tDelIdxGetItem(SDelIdx *pDelIdx, int32_t idx, SDelIdxItem *pItem) { +int32_t tDelIdxGetItem(SDelIdx *pDelIdx, SDelIdxItem *pItem, TABLEID id) { + int32_t code = 0; + int32_t lidx = 0; + int32_t ridx = pDelIdx->nItem - 1; + int32_t midx; + uint64_t offset; + int32_t c; + + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; + + tDelIdxGetItemByIdx(pDelIdx, pItem, midx); + c = tTABLEIDCmprFn(&id, pItem); + if (c == 0) { + goto _exit; + } else if (c < 0) { + ridx = midx - 1; + } else { + lidx = midx + 1; + } + } + + code = TSDB_CODE_NOT_FOUND; + +_exit: + return code; +} + +int32_t tDelIdxGetItemByIdx(SDelIdx *pDelIdx, SDelIdxItem *pItem, int32_t idx) { int32_t code = 0; - // TODO + + tGetDelIdxItem(pDelIdx->pData + tsdbGetOffset(pDelIdx->pOffset, pDelIdx->flags, idx), pItem); + return code; } int32_t tDelIdxPutItem(SDelIdx *pDelIdx, SDelIdxItem *pItem) { - int32_t code = 0; - // TODO + int32_t code = 0; + int32_t size = tPutDelIdxItem(NULL, pItem); + uint32_t offset = pDelIdx->nData; + uint32_t nItem = pDelIdx->nItem; + + pDelIdx->nItem++; + pDelIdx->nData += size; + + // alloc + code = tsdbRealloc(&pDelIdx->pOffset, pDelIdx->nItem * sizeof(uint32_t)); + if (code) goto _exit; + code = tsdbRealloc(&pDelIdx->pData, pDelIdx->nData); + if (code) goto _exit; + + // put + ((uint32_t *)pDelIdx->pOffset)[nItem] = offset; + tPutDelIdxItem(pDelIdx->pData + offset, pItem); + +_exit: return code; } @@ -112,7 +214,8 @@ int32_t tPutDelIdx(uint8_t *p, SDelIdx *pDelIdx) { n += tPutU32(p ? p + n : p, pDelIdx->delimiter); n += tPutU8(p ? p + n : p, pDelIdx->flags); - n += tPutBinary(p ? p + n : p, pDelIdx->pOffset, pDelIdx->nOffset); + n += tPutU32v(p ? p + n : p, pDelIdx->nItem); + n += tPutBinary(p ? p + n : p, pDelIdx->pOffset, pDelIdx->nItem * tsdbOffsetSize(pDelIdx->flags)); n += tPutBinary(p ? p + n : p, pDelIdx->pData, pDelIdx->nData); return n; @@ -123,7 +226,8 @@ int32_t tGetDelIdx(uint8_t *p, SDelIdx *pDelIdx) { n += tGetU32(p, &pDelIdx->delimiter); n += tGetU8(p, &pDelIdx->flags); - n += tGetBinary(p, &pDelIdx->pOffset, &pDelIdx->nOffset); + n += tGetU32v(p, &pDelIdx->nItem); + n += tGetBinary(p, &pDelIdx->pOffset, NULL); n += tGetBinary(p, &pDelIdx->pData, &pDelIdx->nData); return n; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index e122ad0ab6..30e4ff68e6 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp format") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") +TAOS_DEFINE_ERROR(TSDB_CODE_NOT_FOUND, "Not found") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") -- GitLab