From ce9fb842212cd74f1cb54b6c523ed3ffaa197d3b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 13 Sep 2022 14:21:43 +0800 Subject: [PATCH] refact --- include/common/tdataformat.h | 43 ++++- source/common/src/tdataformat.c | 175 +++++++++++++++++- source/dnode/vnode/src/inc/tsdb.h | 26 --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 244 +------------------------ 4 files changed, 210 insertions(+), 278 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index df16f4f0ab..2edecc6199 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -36,8 +36,13 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct STagVal STagVal; typedef struct STag STag; +typedef struct SColData SColData; -// bitmap +#define HAS_NONE ((int8_t)0x1) +#define HAS_NULL ((int8_t)0x2) +#define HAS_VALUE ((int8_t)0x4) + +// bitmap ================================ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00000100, 0b00001000, 2}, {0b00000000, 0b00010000, 0b00100000, 4}, @@ -51,21 +56,21 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) -// STSchema +// STSchema ================================ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); -// SValue +// SValue ================================ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type); int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); -// SColVal +// SColVal ================================ #define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1}) #define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1}) #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)}) -// STSRow2 +// STSRow2 ================================ #define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) @@ -77,7 +82,7 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); -// STSRowBuilder +// STSRowBuilder ================================ #define tsRowBuilderInit() ((STSRowBuilder){0}) #define tsRowBuilderClear(B) \ do { \ @@ -86,7 +91,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); } \ } while (0) -// STag +// STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagIsJson(const void *pTag); @@ -100,7 +105,17 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); -// STRUCT ================= +// SColData ================================ +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); +void tColDataReset(SColData *pColData); +void tColDataClear(void *ph); +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); +int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tPutColData(uint8_t *p, SColData *pColData); +int32_t tGetColData(uint8_t *p, SColData *pColData); + +// STRUCT ================================ struct STColumn { col_id_t colId; int8_t type; @@ -166,6 +181,18 @@ struct SColVal { SValue value; }; +struct SColData { + int16_t cid; + int8_t type; + int8_t smaOn; + int32_t nVal; + uint8_t flag; + uint8_t *pBitMap; + int32_t *aOffset; + int32_t nData; + uint8_t *pData; +}; + #pragma pack(push, 1) struct STagVal { // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b40f449a05..48fb0bb5f4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tdataformat.h" +#include "tRealloc.h" #include "tcoding.h" #include "tdatablock.h" #include "tlog.h" @@ -1172,4 +1173,176 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { return pSchema; } -#endif \ No newline at end of file +#endif + +// SColData ======================================== +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { + pColData->cid = cid; + pColData->type = type; + pColData->smaOn = smaOn; + tColDataReset(pColData); +} + +void tColDataReset(SColData *pColData) { + pColData->nVal = 0; + pColData->flag = 0; + pColData->nData = 0; +} + +void tColDataClear(void *ph) { + SColData *pColData = (SColData *)ph; + + tFree(pColData->pBitMap); + tFree((uint8_t *)pColData->aOffset); + tFree(pColData->pData); +} + +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + int64_t size; + SValue value = {0}; + SValue *pValue = &value; + + ASSERT(pColVal->cid == pColData->cid); + ASSERT(pColVal->type == pColData->type); + + // realloc bitmap + size = BIT2_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, size); + if (code) goto _exit; + if ((pColData->nVal & 3) == 0) { + pColData->pBitMap[pColData->nVal >> 2] = 0; + } + + // put value + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + } else if (pColVal->isNull) { + pColData->flag |= HAS_NULL; + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + } else { + pColData->flag |= HAS_VALUE; + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + pValue = &pColVal->value; + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { + // offset + code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1)); + if (code) goto _exit; + pColData->aOffset[pColData->nVal] = pColData->nData; + + // value + if ((!pColVal->isNone) && (!pColVal->isNull)) { + code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (code) goto _exit; + memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); + pColData->nData += pColVal->value.nData; + } + } else { + code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type)); + if (code) goto _exit; + pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type); + } + + pColData->nVal++; + +_exit: + return code; +} + +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { + int32_t code = 0; + int32_t size; + + ASSERT(pColDataSrc->nVal > 0); + ASSERT(pColDataDest->cid = pColDataSrc->cid); + ASSERT(pColDataDest->type = pColDataSrc->type); + + pColDataDest->smaOn = pColDataSrc->smaOn; + pColDataDest->nVal = pColDataSrc->nVal; + pColDataDest->flag = pColDataSrc->flag; + + // bitmap + if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { + size = BIT2_SIZE(pColDataSrc->nVal); + code = tRealloc(&pColDataDest->pBitMap, size); + if (code) goto _exit; + memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + } + + // offset + if (IS_VAR_DATA_TYPE(pColDataDest->type)) { + size = sizeof(int32_t) * pColDataSrc->nVal; + + code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); + if (code) goto _exit; + + memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + } + + // value + pColDataDest->nData = pColDataSrc->nData; + code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); + if (code) goto _exit; + memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); + +_exit: + return code; +} + +int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { + int32_t code = 0; + + ASSERT(iVal < pColData->nVal); + ASSERT(pColData->flag); + + if (pColData->flag == HAS_NONE) { + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + goto _exit; + } else if (pColData->flag == HAS_NULL) { + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + goto _exit; + } else if (pColData->flag != HAS_VALUE) { + uint8_t v = GET_BIT2(pColData->pBitMap, iVal); + if (v == 0) { + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + goto _exit; + } else if (v == 1) { + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + goto _exit; + } + } + + // get value + SValue value; + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iVal + 1 < pColData->nVal) { + value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; + } else { + value.nData = pColData->nData - pColData->aOffset[iVal]; + } + + value.pData = pColData->pData + pColData->aOffset[iVal]; + } else { + tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + } + *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); + +_exit: + return code; +} + +static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { + SColData *pColData1 = (SColData *)p1; + SColData *pColData2 = (SColData *)p2; + + if (pColData1->cid < pColData2->cid) { + return -1; + } else if (pColData1->cid > pColData2->cid) { + return 1; + } + + return 0; +} diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a836fa2bc5..063b33f3a9 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -44,7 +44,6 @@ typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; typedef struct SDataBlk SDataBlk; typedef struct SSttBlk SSttBlk; -typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDelFile SDelFile; @@ -71,10 +70,6 @@ typedef struct SLDataIter SLDataIter; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 -#define HAS_NONE ((int8_t)0x1) -#define HAS_NULL ((int8_t)0x2) -#define HAS_VALUE ((int8_t)0x4) - #define VERSION_MIN 0 #define VERSION_MAX INT64_MAX @@ -148,15 +143,6 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); -// SColdata -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); -void tColDataReset(SColData *pColData); -void tColDataClear(void *ph); -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); -int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); -int32_t tPutColData(uint8_t *p, SColData *pColData); -int32_t tGetColData(uint8_t *p, SColData *pColData); // SBlockData #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) @@ -470,18 +456,6 @@ struct SSttBlk { SBlockInfo bInfo; }; -struct SColData { - int16_t cid; - int8_t type; - int8_t smaOn; - int32_t nVal; - uint8_t flag; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; -}; - // (SBlockData){.suid = 0, .uid = 0}: block data not initialized // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index caeca45e01..1a109b2cf6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -909,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr return code; } -// SColData ======================================== -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { - pColData->cid = cid; - pColData->type = type; - pColData->smaOn = smaOn; - tColDataReset(pColData); -} - -void tColDataReset(SColData *pColData) { - pColData->nVal = 0; - pColData->flag = 0; - pColData->nData = 0; -} - -void tColDataClear(void *ph) { - SColData *pColData = (SColData *)ph; - - tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); - tFree(pColData->pData); -} - -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; - int64_t size; - SValue value = {0}; - SValue *pValue = &value; - - ASSERT(pColVal->cid == pColData->cid); - ASSERT(pColVal->type == pColData->type); - - // realloc bitmap - size = BIT2_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, size); - if (code) goto _exit; - if ((pColData->nVal & 3) == 0) { - pColData->pBitMap[pColData->nVal >> 2] = 0; - } - - // put value - if (pColVal->isNone) { - pColData->flag |= HAS_NONE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (pColVal->isNull) { - pColData->flag |= HAS_NULL; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - pValue = &pColVal->value; - } - - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1)); - if (code) goto _exit; - pColData->aOffset[pColData->nVal] = pColData->nData; - - // value - if ((!pColVal->isNone) && (!pColVal->isNull)) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); - if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; - } - } else { - code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type)); - if (code) goto _exit; - pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type); - } - - pColData->nVal++; - -_exit: - return code; -} - -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { - int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid = pColDataSrc->cid); - ASSERT(pColDataDest->type = pColDataSrc->type); - - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; - - // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); - } - - // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; - - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; - - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); - } - - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); - -_exit: - return code; -} - -int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { - int32_t code = 0; - - ASSERT(iVal < pColData->nVal); - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag == HAS_NULL) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag != HAS_VALUE) { - uint8_t v = GET_BIT2(pColData->pBitMap, iVal); - if (v == 0) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (v == 1) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } - } - - // get value - SValue value; - if (IS_VAR_DATA_TYPE(pColData->type)) { - if (iVal + 1 < pColData->nVal) { - value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; - } else { - value.nData = pColData->nData - pColData->aOffset[iVal]; - } - - value.pData = pColData->pData + pColData->aOffset[iVal]; - } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); - } - *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); - -_exit: - return code; -} - -int32_t tPutColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tPutI16v(p ? p + n : p, pColData->cid); - n += tPutI8(p ? p + n : p, pColData->type); - n += tPutI8(p ? p + n : p, pColData->smaOn); - n += tPutI32v(p ? p + n : p, pColData->nVal); - n += tPutU8(p ? p + n : p, pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - if (p) { - memcpy(p + n, pColData->pBitMap, size); - } - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - if (p) { - memcpy(p + n, pColData->aOffset, size); - } - n += size; - } - n += tPutI32v(p ? p + n : p, pColData->nData); - if (p) { - memcpy(p + n, pColData->pData, pColData->nData); - } - n += pColData->nData; - -_exit: - return n; -} - -int32_t tGetColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tGetI16v(p + n, &pColData->cid); - n += tGetI8(p + n, &pColData->type); - n += tGetI8(p + n, &pColData->smaOn); - n += tGetI32v(p + n, &pColData->nVal); - n += tGetU8(p + n, &pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - pColData->pBitMap = p + n; - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - pColData->aOffset = (int32_t *)(p + n); - n += size; - } - n += tGetI32v(p + n, &pColData->nData); - pColData->pData = p + n; - n += pColData->nData; - -_exit: - return n; -} - -static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { - SColData *pColData1 = (SColData *)p1; - SColData *pColData2 = (SColData *)p2; - - if (pColData1->cid < pColData2->cid) { - return -1; - } else if (pColData1->cid > pColData2->cid) { - return 1; - } - - return 0; -} - // SBlockData ====================================================== int32_t tBlockDataCreate(SBlockData *pBlockData) { int32_t code = 0; @@ -1501,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD while (lidx <= ridx) { int32_t midx = (lidx + ridx) / 2; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); - int32_t c = tColDataCmprFn(pColData, &(SColData){.cid = cid}); + int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); if (c == 0) { *ppColData = pColData; -- GitLab