From 37239ffaea1f006080328ca8f6c25bdb4b9679a8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 28 Nov 2022 15:37:47 +0800 Subject: [PATCH] more code --- include/common/tdataformat.h | 1 + source/common/src/tdataformat.c | 116 +++++++++- source/dnode/vnode/src/tsdb/tsdbUtil.c | 302 ++++++++++++------------- 3 files changed, 254 insertions(+), 165 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 60acb78140..f3c1bc22c2 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -93,6 +93,7 @@ int32_t tRowMergeSort(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); void tRowIterClose(SRowIter **ppIter); SColVal *tRowIterNext(SRowIter *pIter); +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData); // STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c767378ed6..f1054eefce 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -20,6 +20,8 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); + // SBuffer ================================ void tBufferDestroy(SBuffer *pBuffer) { tFree(pBuffer->pBuf); @@ -839,7 +841,96 @@ _exit: return &pIter->cv; } -// STSchema ======================================== +static int32_t tRowAppendNoneToColData(SArray *aColData, int32_t nColData) { + int32_t code = 0; + + for (int32_t iColData = 0; iColData < nColData; iColData++) { + SColData *pColData = taosArrayGet(aColData, iColData); + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + } + +_exit: + return code; +} +static int32_t tRowAppendNullToColData(SArray *aColData, int32_t nColData, STSchema *pSchema) { + int32_t code = 0; + + int32_t iColData = 0; + SColData *pColData = taosArrayGet(aColData, iColData); + int32_t iTColumn = 1; + STColumn *pTColumn = &pSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { // NULL + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL; + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL; + } else { + pTColumn = (++iTColumn < pSchema->numOfCols) ? &pSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL; + } + } + +_exit: + return code; +} +int32_t tRowAppendToColData(SRow *pRow, STSchema *pTSchema, SArray *aColData, int32_t nColData) { + ASSERT(pRow->sver == pTSchema->version); + ASSERT(nColData > 0); + + int32_t code = 0; + + if (pRow->flag == HAS_NONE) { + code = tRowAppendNoneToColData(aColData, nColData); + goto _exit; + } else if (pRow->flag == HAS_NULL) { + code = tRowAppendNullToColData(aColData, nColData, pTSchema); + goto _exit; + } + + if (pRow->flag >> 4) { // KV row + // todo + } else { // Tuple row + int32_t iColData = 0; + SColData *pColData = taosArrayGet(aColData, iColData); + int32_t iTColumn = 1; + STColumn *pTColumn = &pTSchema->columns[iTColumn]; + + while (pColData) { + if (pTColumn) { + if (pTColumn->colId == pColData->cid) { + } else if (pTColumn->colId > pColData->cid) { // NONE + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL; + } else { + pTColumn = (++iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + } + } else { + code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); + if (code) goto _exit; + + pColData = (++iColData < nColData) ? taosArrayGet(aColData, iColData) : NULL; + } + } + } + +_exit: + return code; +} // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { @@ -1217,6 +1308,7 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { tPutI16v(p + offset, cid); } +// STSchema ======================================== STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); if (pTSchema == NULL) return NULL; @@ -1305,7 +1397,11 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, uint8_t *pData, ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pData, tDataTypes[pColData->type].bytes); + if (pData) { + memcpy(pColData->pData + pColData->nData, pData, TYPE_BYTES[pColData->type]); + } else { + memset(pColData->pData + pColData->nData, 0, TYPE_BYTES[pColData->type]); + } pColData->nData += tDataTypes[pColData->type].bytes; } pColData->nVal++; @@ -1489,7 +1585,7 @@ static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, uint8_t *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1503,7 +1599,7 @@ static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, uint8_t *p memset(pColData->pBitMap, 255, nBit); SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1523,7 +1619,7 @@ static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, uint8_t *p SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1542,7 +1638,7 @@ static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, uint8_t *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1570,7 +1666,7 @@ static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, uint8_t *p tFree(pColData->pBitMap); pColData->pBitMap = pBitMap; - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1579,7 +1675,7 @@ static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, uint8_t *p if (code) return code; SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1597,7 +1693,7 @@ static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, uint8_t *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *pData, uint32_t nData) { int32_t code = 0; @@ -1606,7 +1702,7 @@ static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, uint8_t *p if (code) return code; SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - return tColDataPutValue(pColData, pData, nData); + return tColDataPutValue(pColData, NULL, 0); } static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData) = { {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1f02a9a3ef..120e9193b8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1073,152 +1073,152 @@ _exit: return code; } -static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - - if (pRow->statis) { - TDRowValT vt = TD_VTYPE_MAX; - tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - } else { - cv.flag = CV_FLAG_VALUE; - - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } - -_exit: - return code; -} - -static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { - int32_t code = 0; - - col_id_t kvIter = 0; - col_id_t nKvCols = tdRowGetNCols(pRow) - 1; - void *pColIdx = TD_ROW_COL_IDX(pRow); - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - int32_t iTColumn = 1; - STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - - for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { - SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; - - while (pTColumn && pTColumn->colId < pColData->cid) { - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - - if (pTColumn == NULL || pTColumn->colId > pColData->cid) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(pTColumn->type == pColData->type); - - SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; - TDRowValT vt = TD_VTYPE_NONE; // default is NONE - SKvRowIdx *pKvIdx = NULL; - - while (kvIter < nKvCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); - if (pKvIdx->colId == pTColumn->colId) { - tdGetBitmapValTypeII(pBitmap, kvIter, &vt); - ++kvIter; - break; - } else if (pKvIdx->colId > pTColumn->colId) { - vt = TD_VTYPE_NONE; - break; - } else { - ++kvIter; - } - } - - if (vt == TD_VTYPE_NORM) { - cv.flag = CV_FLAG_VALUE; - - void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); - if (IS_VAR_DATA_TYPE(pTColumn->type)) { - cv.value.nData = varDataLen(pData); - cv.value.pData = varDataVal(pData); - } else { - memcpy(&cv.value.val, pData, pTColumn->bytes); - } - - code = tColDataAppendValue(pColData, &cv); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NONE) { - code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); - if (code) goto _exit; - } else if (vt == TD_VTYPE_NULL) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); - if (code) goto _exit; - } else { - ASSERT(0); - } - - iTColumn++; - pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; - } - } +// static int32_t tBlockDataAppendTPRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { +// int32_t code = 0; + +// int32_t iTColumn = 1; +// STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; +// void *pBitmap = pRow->statis ? tdGetBitmapAddrTp(pRow, pTSchema->flen) : NULL; + +// for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { +// SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; + +// while (pTColumn && pTColumn->colId < pColData->cid) { +// iTColumn++; +// pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; +// } + +// if (pTColumn == NULL || pTColumn->colId > pColData->cid) { +// code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else { +// ASSERT(pTColumn->type == pColData->type); + +// SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; + +// if (pRow->statis) { +// TDRowValT vt = TD_VTYPE_MAX; +// tdGetBitmapValTypeII(pBitmap, iTColumn - 1, &vt); + +// if (vt == TD_VTYPE_NORM) { +// cv.flag = CV_FLAG_VALUE; + +// if (IS_VAR_DATA_TYPE(pTColumn->type)) { +// void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); +// cv.value.nData = varDataLen(pData); +// cv.value.pData = varDataVal(pData); +// } else { +// memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); +// } + +// code = tColDataAppendValue(pColData, &cv); +// if (code) goto _exit; +// } else if (vt == TD_VTYPE_NONE) { +// code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else if (vt == TD_VTYPE_NULL) { +// code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else { +// ASSERT(0); +// } +// } else { +// cv.flag = CV_FLAG_VALUE; + +// if (IS_VAR_DATA_TYPE(pTColumn->type)) { +// void *pData = (char *)pRow + *(int32_t *)(pRow->data + pTColumn->offset); +// cv.value.nData = varDataLen(pData); +// cv.value.pData = varDataVal(pData); +// } else { +// memcpy(&cv.value.val, pRow->data + pTColumn->offset, pTColumn->bytes); +// } + +// code = tColDataAppendValue(pColData, &cv); +// if (code) goto _exit; +// } + +// iTColumn++; +// pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; +// } +// } + +// _exit: +// return code; +// } -_exit: - return code; -} +// static int32_t tBlockDataAppendKVRow(SBlockData *pBlockData, STSRow *pRow, STSchema *pTSchema) { +// int32_t code = 0; + +// col_id_t kvIter = 0; +// col_id_t nKvCols = tdRowGetNCols(pRow) - 1; +// void *pColIdx = TD_ROW_COL_IDX(pRow); +// void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); +// int32_t iTColumn = 1; +// STColumn *pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; + +// for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { +// SColData *pColData = &((SColData *)pBlockData->aColData->pData)[iColData]; + +// while (pTColumn && pTColumn->colId < pColData->cid) { +// iTColumn++; +// pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; +// } + +// if (pTColumn == NULL || pTColumn->colId > pColData->cid) { +// code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else { +// ASSERT(pTColumn->type == pColData->type); + +// SColVal cv = {.cid = pTColumn->colId, .type = pTColumn->type}; +// TDRowValT vt = TD_VTYPE_NONE; // default is NONE +// SKvRowIdx *pKvIdx = NULL; + +// while (kvIter < nKvCols) { +// pKvIdx = (SKvRowIdx *)POINTER_SHIFT(pColIdx, kvIter * sizeof(SKvRowIdx)); +// if (pKvIdx->colId == pTColumn->colId) { +// tdGetBitmapValTypeII(pBitmap, kvIter, &vt); +// ++kvIter; +// break; +// } else if (pKvIdx->colId > pTColumn->colId) { +// vt = TD_VTYPE_NONE; +// break; +// } else { +// ++kvIter; +// } +// } + +// if (vt == TD_VTYPE_NORM) { +// cv.flag = CV_FLAG_VALUE; + +// void *pData = POINTER_SHIFT(pRow, pKvIdx->offset); +// if (IS_VAR_DATA_TYPE(pTColumn->type)) { +// cv.value.nData = varDataLen(pData); +// cv.value.pData = varDataVal(pData); +// } else { +// memcpy(&cv.value.val, pData, pTColumn->bytes); +// } + +// code = tColDataAppendValue(pColData, &cv); +// if (code) goto _exit; +// } else if (vt == TD_VTYPE_NONE) { +// code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else if (vt == TD_VTYPE_NULL) { +// code = tColDataAppendValue(pColData, &COL_VAL_NULL(pColData->cid, pColData->type)); +// if (code) goto _exit; +// } else { +// ASSERT(0); +// } + +// iTColumn++; +// pTColumn = (iTColumn < pTSchema->numOfCols) ? &pTSchema->columns[iTColumn] : NULL; +// } +// } + +// _exit: +// return code; +// } int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { int32_t code = 0; @@ -1243,16 +1243,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS SColVal cv = {0}; if (pRow->type == 0) { - ASSERT(0); - // if (TD_IS_TP_ROW(pRow->pTSRow)) { - // code = tBlockDataAppendTPRow(pBlockData, pRow->pTSRow, pTSchema); - // if (code) goto _err; - // } else if (TD_IS_KV_ROW(pRow->pTSRow)) { - // code = tBlockDataAppendKVRow(pBlockData, pRow->pTSRow, pTSchema); - // if (code) goto _err; - // } else { - // ASSERT(0); - // } + code = tRowAppendToColData(pRow->pTSRow, pTSchema, pBlockData->aColData, pBlockData->nColData); + if (code) goto _err; } else { code = tBlockDataAppendBlockRow(pBlockData, pRow->pBlockData, pRow->iRow); if (code) goto _err; -- GitLab