From 76a9ef186efde3a66cd194abef6f558a1bd48bac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 9 May 2022 13:10:16 +0000 Subject: [PATCH] refact row apis --- include/common/trow.h | 1159 ++------------------------------------ source/common/src/trow.c | 934 ++++++++++++++++++++++++++++++ 2 files changed, 975 insertions(+), 1118 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index ebce0b085d..eed15689c0 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -95,8 +95,6 @@ static FORCE_INLINE bool tdValIsNull(TDRowValT valType, const void *val, int32_t #endif } -typedef void *SRow; - typedef struct { TDRowValT valType; void *val; @@ -228,24 +226,19 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode, bool isMerge); -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool isMerge); + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); /** * @brief @@ -264,257 +257,11 @@ static FORCE_INLINE void *tdGetBitmapAddrKv(STSRow *pRow, col_id_t nKvCols) { // The primary TS key is stored separatedly and is Norm value, thus should minus 1 firstly return POINTER_SHIFT(TD_ROW_COL_IDX(pRow), (--nKvCols) * sizeof(SKvRowIdx)); } -static FORCE_INLINE void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { -#ifdef TD_SUPPORT_BITMAP - switch (rowType) { - case TD_ROW_TP: - return tdGetBitmapAddrTp(pRow, flen); - case TD_ROW_KV: - return tdGetBitmapAddrKv(pRow, nKvCols); - default: - break; - } -#endif - return NULL; -} - -static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdSetBitmapValTypeII(pBitmap, colIdx, valType); - break; - case -1: - case 1: - tdSetBitmapValTypeI(pBitmap, colIdx, valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 6); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xFC) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { - TDRowValT valType = 0; - tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); - if (tdValTypeIsNorm(valType)) { - return true; - } - return false; -} - -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, - int8_t bitmapMode) { - switch (bitmapMode) { - case 0: - tdGetBitmapValTypeII(pBitmap, colIdx, pValType); - break; - case -1: - case 1: - tdGetBitmapValTypeI(pBitmap, colIdx, pValType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return TSDB_CODE_FAILED; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief Use 2 bits at default - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS; - int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0xC0) >> 6); - break; - case 1: - *pValType = (((*pDestByte) & 0x30) >> 4); - break; - case 2: - *pValType = (((*pDestByte) & 0x0C) >> 2); - break; - case 3: - *pValType = ((*pDestByte) & 0x03); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param valType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); - // set the value and clear other partitions for offset 0 - // *pDestByte |= (valType << 7); - break; - case 1: - *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); - // *pDestByte |= (valType << 6); - break; - case 2: - *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); - // *pDestByte |= (valType << 5); - break; - case 3: - *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); - // *pDestByte |= (valType << 4); - break; - case 4: - *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); - // *pDestByte |= (valType << 3); - break; - case 5: - *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); - // *pDestByte |= (valType << 2); - break; - case 6: - *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); - // *pDestByte |= (valType << 1); - break; - case 7: - *pDestByte = ((*pDestByte) & 0xFE) | valType; - // *pDestByte |= (valType); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBitmap - * @param colIdx The relative index of colId, may have minus value as parameter. - * @param pValType - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { - if (!pBitmap || colIdx < 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; - int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; - char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); - // use literal value directly and not use formula to simplify the codes - switch (nOffset) { - case 0: - *pValType = (((*pDestByte) & 0x80) >> 7); - break; - case 1: - *pValType = (((*pDestByte) & 0x40) >> 6); - break; - case 2: - *pValType = (((*pDestByte) & 0x20) >> 5); - break; - case 3: - *pValType = (((*pDestByte) & 0x10) >> 4); - break; - case 4: - *pValType = (((*pDestByte) & 0x08) >> 3); - break; - case 5: - *pValType = (((*pDestByte) & 0x04) >> 2); - break; - case 6: - *pValType = (((*pDestByte) & 0x02) >> 1); - break; - case 7: - *pValType = ((*pDestByte) & 0x01); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols); +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); // ----------------- Tuple row structure(STpRow) /* @@ -539,482 +286,28 @@ static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t col * */ -/** - * @brief - * - * @param pBuilder - * @param sver schema version - * @return FORCE_INLINE - */ static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { pBuilder->rowType = TD_ROW_TP; // default STpRow pBuilder->sver = sver; } - -/** - * @brief Not recommended to use - * - * @param pBuilder - * @param rowType - * @return FORCE_INLINE - */ -static FORCE_INLINE void tdSRowSetRowType(SRowBuilder *pBuilder, int8_t rowType) { pBuilder->rowType = rowType; } - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pBuilder - * @param nCols - * @param nBoundCols use -1 if not available - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { - pBuilder->flen = flen; - pBuilder->nCols = nCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief To judge row type: STpRow/SKvRow - * - * @param pBuilder - * @param nCols - * @param nBoundCols - * @param flen - * @param allNullLen use -1 if not available - * @param boundNullLen use -1 if not available - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, - int32_t flen, int32_t allNullLen, int32_t boundNullLen) { - if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { - uint32_t tpLen = allNullLen; - uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; - if (isSelectKVRow(kvLen, tpLen)) { - pBuilder->rowType = TD_ROW_KV; - } else { - pBuilder->rowType = TD_ROW_TP; - } - - } else { - pBuilder->rowType = TD_ROW_TP; - } - - pBuilder->flen = flen; - pBuilder->nCols = nCols; - pBuilder->nBoundCols = nBoundCols; - if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } -#ifdef TD_SUPPORT_BITMAP - // the primary TS key is stored separatedly - pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); - if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); - } else { - pBuilder->nBoundBitmaps = 0; - } -#else - pBuilder->nBitmaps = 0; - pBuilder->nBoundBitmaps = 0; -#endif - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TD_ROW_SET_INFO(pBuilder->pBuf, 0); - TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); -#endif - // the primary TS key is stored separatedly - len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); - memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); -#endif - len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + - pBuilder->nBoundBitmaps; // add - TD_ROW_SET_LEN(pBuilder->pBuf, len); - TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); - TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - return TSDB_CODE_SUCCESS; -} - -/** - * @brief The invoker is responsible for memory alloc/dealloc. - * - * @param pBuilder - * @param pBuf Output buffer of STSRow - */ -static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { - pBuilder->pBuf = (STSRow *)pBuf; - if (!pBuilder->pBuf) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); - - uint32_t len = 0; - switch (pBuilder->rowType) { - case TD_ROW_TP: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); -#endif - break; - case TD_ROW_KV: -#ifdef TD_SUPPORT_BITMAP - pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); -#endif - break; - default: - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief 由调用方管理存储空间的分配及释放，一次输入多个参数 - * - * @param pBuilder - * @param pBuf - * @param allNullLen - * @param boundNullLen - * @param nCols - * @param nBoundCols - * @param flen - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, - int32_t nCols, int32_t nBoundCols, int32_t flen) { - if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { - return terrno; - } - return tdSRowResetBuf(pBuilder, pBuf); -} - -/** - * @brief - * - * @param pBuilder - */ -static FORCE_INLINE void tdSRowReset(SRowBuilder *pBuilder) { - pBuilder->rowType = TD_ROW_TP; - pBuilder->pBuf = NULL; - pBuilder->nBoundCols = -1; - pBuilder->nCols = -1; - pBuilder->flen = -1; - pBuilder->pBitmap = NULL; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset) { - if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(TSKEY); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - - // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap - // should be updated simultaneously if Norm val overwrite Null/None cols. - // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. - if (tdValIsNorm(valType, val, colType)) { - // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - // TODO: Null value for new data types imported since 3.0 need to be defined. - const void *nullVal = getNullValue(colType); - if (IS_VAR_DATA_TYPE(colType)) { - // ts key stored in STSRow.ts - *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); - - if (isCopyVarData) { - memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); - } - } -#endif - - return 0; -} - -// internal func -static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, - bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, - col_id_t colId) { - if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - offset -= sizeof(SKvRowIdx); - --colIdx; - -#ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { - return terrno; - } -#endif - - STSRow *row = pBuilder->pBuf; - // No need to store None/Null values. - if (tdValIsNorm(valType, val, colType)) { - // ts key stored in STSRow.ts - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, val, varDataTLen(val)); - } - TD_ROW_LEN(row) += varDataTLen(val); - } else { - memcpy(ptr, val, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#ifdef TD_SUPPORT_BACK2 - // NULL/None value - else { - SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); - pColIdx->colId = colId; - pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN - const void *nullVal = getNullValue(colType); - - if (IS_VAR_DATA_TYPE(colType)) { - if (isCopyVarData) { - memcpy(ptr, nullVal, varDataTLen(nullVal)); - } - TD_ROW_LEN(row) += varDataTLen(nullVal); - } else { - memcpy(ptr, nullVal, TYPE_BYTES[colType]); - TD_ROW_LEN(row) += TYPE_BYTES[colType]; - } - } -#endif - - return 0; -} - -/** - * @brief exposed func - * - * @param pBuilder - * @param colId start from PRIMARYKEY_TIMESTAMP_COL_ID - * @param colType - * @param valType - * @param val - * @param isCopyVarData - * @param offset - * @param colIdx sorted column index, start from 0 - * @return FORCE_INLINE - */ -static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, - TDRowValT valType, const void *val, bool isCopyVarData, int32_t offset, - col_id_t colIdx) { - STSRow *pRow = pBuilder->pBuf; - if (!val) { -#ifdef TD_SUPPORT_BITMAP - if (tdValTypeIsNorm(valType)) { - terrno = TSDB_CODE_INVALID_PTR; - return terrno; - } -#else - TASSERT(0); - terrno = TSDB_CODE_INVALID_PARA; - return terrno; -#endif - } - // TS KEY is stored in STSRow.ts and not included in STSRow.data field. - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - TD_ROW_KEY(pRow) = *(TSKEY *)val; - // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. - return TSDB_CODE_SUCCESS; - } - // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. - if (TD_IS_TP_ROW(pRow)) { - tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); - } else { - tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); - } - return TSDB_CODE_SUCCESS; -} - -// internal -static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, - int32_t offset, int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - } -#else - if (IS_VAR_DATA_TYPE(colType)) { - output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} - -static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { - return -1; - } else { - return 0; - } -} -// internal -static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, - int16_t colIdx) { -#ifdef TD_SUPPORT_BITMAP - TASSERT(colIdx < tdRowGetNCols(pRow) - 1); - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { - output->valType = TD_VTYPE_NONE; - return terrno; - } - if (tdValTypeIsNorm(output->valType)) { - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - } -#else - TASSERT(0); - if (offset < 0) { - terrno = TSDB_CODE_INVALID_PARA; - output->valType = TD_VTYPE_NONE; - return terrno; - } - output->val = POINTER_SHIFT(pRow, offset); - output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - return TSDB_CODE_SUCCESS; -} +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen); +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen); +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf); +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen); +void tdSRowReset(SRowBuilder *pBuilder); +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset); +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx); +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx); +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx); typedef struct { STSchema *pSchema; @@ -1026,390 +319,20 @@ typedef struct { col_id_t kvIdx; // [0, nKvCols) } STSRowIter; -static FORCE_INLINE void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { - pIter->pRow = pRow; - pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); - pIter->offset = 0; - pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; - pIter->kvIdx = 0; -} - -static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { - pIter->pSchema = pSchema; - pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; -} - -static int32_t tdCompareColId(const void *arg1, const void *arg2) { - int32_t colId = *(int32_t *)arg1; - STColumn *pCol = (STColumn *)arg2; - - if (colId < pCol->colId) { - return -1; - } else if (colId == pCol->colId) { - return 0; - } else { - return 1; - } -} - -/** - * @brief STSRow method to get value of specified colId/colType by bsearch - * - * @param pIter - * @param colId Start from PRIMARYKEY_TIMESTAMP_COL_ID(1) - * @param colType - * @param pVal - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - STSRow *pRow = pIter->pRow; - int16_t colIdx = -1; - if (TD_IS_TP_ROW(pRow)) { - STSchema *pSchema = pIter->pSchema; - STColumn *pCol = - (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); - if (!pCol) { - pVal->valType = TD_VTYPE_NONE; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - return true; - } -#ifdef TD_SUPPORT_BITMAP - colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); -#endif - tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); - } else if (TD_IS_KV_ROW(pRow)) { - SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), - compareKvRowColId, TD_EQ); -#ifdef TD_SUPPORT_BITMAP - if (pIdx) { - colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); - } -#endif - tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); - } else { - if (COL_REACH_END(colId, pIter->maxColId)) return false; - pVal->valType = TD_VTYPE_NONE; - } - - return true; -} - -// internal -static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - if (IS_VAR_DATA_TYPE(colType)) { - pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); - } else { - pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); - } - -#ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -// internal -static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, - SCellVal *pVal) { - STSRow *pRow = pIter->pRow; - SKvRowIdx *pKvIdx = NULL; - bool colFound = false; - col_id_t kvNCols = tdRowGetNCols(pRow) - 1; - while (*nIdx < kvNCols) { - pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); - if (pKvIdx->colId == colId) { - ++(*nIdx); - pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); - colFound = true; - break; - } else if (pKvIdx->colId > colId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - ++(*nIdx); - } - } - - if (!colFound) { - if (colId <= pIter->maxColId) { - pVal->valType = TD_VTYPE_NONE; - return true; - } else { - return false; - } - } - -#ifdef TD_SUPPORT_BITMAP - int16_t colIdx = -1; - if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { - pVal->valType = TD_VTYPE_NONE; - } -#else - pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; -#endif - - return true; -} - -/** - * @brief STSRow Iter to get value from colId 1 to maxColId ascendingly - * - * @param pIter - * @param pVal - * @param colId - * @param colType - * @param pVal output - * @return true Not reach end and pVal is set(None/Null/Norm). - * @return false Reach end of row and pVal not set. - */ -static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - pVal->val = &pIter->pRow->ts; - pVal->valType = TD_VTYPE_NORM; - return true; - } - - if (TD_IS_TP_ROW(pIter->pRow)) { - STColumn *pCol = NULL; - STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx < pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key - if (colId == pCol->colId) { - break; - } else if (pCol->colId < colId) { - ++pIter->colIdx; - continue; - } else { - return false; - } - } - tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); - ++pIter->colIdx; - } else if (TD_IS_KV_ROW(pIter->pRow)) { - return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); - } else { - pVal->valType = TD_VTYPE_NONE; - terrno = TSDB_CODE_INVALID_PARA; - if (COL_REACH_END(colId, pIter->maxColId)) return false; - } - return true; -} - +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow); +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema); +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal); +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal); +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal); STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); - -// Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { - if (isAllRowsNone(pCol)) { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - return TSDB_CODE_SUCCESS; - } - - if (TD_COL_ROWS_NORM(pCol)) { - pVal->valType = TD_VTYPE_NORM; - } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { - return terrno; - } - - if (tdValTypeIsNorm(pVal->valType)) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - } else { - pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - } - } else { - pVal->valType = TD_VTYPE_NULL; -#ifdef TD_SUPPORT_READ2 - pVal->val = (void *)getNullValue(pCol->type); -#else - pVal->val = NULL; -#endif - } - return TSDB_CODE_SUCCESS; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, - col_id_t colIdx, SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrTp(pRow, flen); - tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); - return true; -} - -/** - * @brief - * - * @param pRow - * @param colId - * @param offset - * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE - */ -static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, - SCellVal *pVal) { - if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); - return true; - } - void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); - tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); - return true; -} - -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { - ASSERT(rows > 0); - int32_t result = 0; - - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - result += pDataCol->dataOff[rows - 1]; - SCellVal val = {0}; - if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { - TASSERT(0); - } - - // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 - result += varDataTLen(val.val); - // TODO: later on, don't save Null/None for VarDataT for 3.0 - // if (tdValTypeIsNorm(val.valType)) { - // result += varDataTLen(val.val); - // } - } else { - result += TYPE_BYTES[pDataCol->type] * rows; - } - - ASSERT(pDataCol->len == result); - - return result; -} - -static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { - if (tdValTypeIsNull(pVal->valType)) { - printf("NULL "); - return; - } else if (tdValTypeIsNone(pVal->valType)) { - printf("NONE "); - return; - } - switch (colType) { - case TSDB_DATA_TYPE_BOOL: - printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); - break; - case TSDB_DATA_TYPE_TINYINT: - printf("%" PRIi8 " ", *(int8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_SMALLINT: - printf("%" PRIi16 " ", *(int16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_INT: - printf("%" PRIi32 " ", *(int32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_BIGINT: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_FLOAT: - printf("%f ", *(float *)pVal->val); - break; - case TSDB_DATA_TYPE_DOUBLE: - printf("%lf ", *(double *)pVal->val); - break; - case TSDB_DATA_TYPE_VARCHAR: - printf("VARCHAR "); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - printf("%" PRIi64 " ", *(int64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_NCHAR: - printf("NCHAR "); - break; - case TSDB_DATA_TYPE_UTINYINT: - printf("%" PRIu8 " ", *(uint8_t *)pVal->val); - break; - case TSDB_DATA_TYPE_USMALLINT: - printf("%" PRIu16 " ", *(uint16_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UINT: - printf("%" PRIu32 " ", *(uint32_t *)pVal->val); - break; - case TSDB_DATA_TYPE_UBIGINT: - printf("%" PRIu64 " ", *(uint64_t *)pVal->val); - break; - case TSDB_DATA_TYPE_JSON: - printf("JSON "); - break; - case TSDB_DATA_TYPE_VARBINARY: - printf("VARBIN "); - break; - case TSDB_DATA_TYPE_DECIMAL: - printf("DECIMAL "); - break; - case TSDB_DATA_TYPE_BLOB: - printf("BLOB "); - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - printf("MedBLOB "); - break; - // case TSDB_DATA_TYPE_BINARY: - // printf("BINARY "); - // break; - case TSDB_DATA_TYPE_MAX: - printf("UNDEF "); - break; - default: - printf("UNDEF "); - break; - } -} - -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pSchema); - tdSTSRowIterReset(&iter, row); - printf("%s >>>", tag); - for (int i = 0; i < pSchema->numOfCols; ++i) { - STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {255, NULL}; - if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { - break; - } - ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); - tdSCellValPrint(&sVal, stCol->type); - } - printf("\n"); -} +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode); +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal); +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal); +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode); +void tdSCellValPrint(SCellVal *pVal, int8_t colType); +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag); #ifdef __cplusplus } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 50e51a40dd..44bcd72a33 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -33,6 +33,7 @@ const uint8_t tdVTypeByte[2][3] = {{ // declaration static uint8_t tdGetBitmapByte(uint8_t byte); +static int32_t tdCompareColId(const void *arg1, const void *arg2); // static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -916,4 +917,937 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { } return pRet; +} + +void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pSchema); + tdSTSRowIterReset(&iter, row); + printf("%s >>>", tag); + for (int i = 0; i < pSchema->numOfCols; ++i) { + STColumn *stCol = pSchema->columns + i; + SCellVal sVal = {255, NULL}; + if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + break; + } + ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); + tdSCellValPrint(&sVal, stCol->type); + } + printf("\n"); +} + +void tdSCellValPrint(SCellVal *pVal, int8_t colType) { + if (tdValTypeIsNull(pVal->valType)) { + printf("NULL "); + return; + } else if (tdValTypeIsNone(pVal->valType)) { + printf("NONE "); + return; + } + switch (colType) { + case TSDB_DATA_TYPE_BOOL: + printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%" PRIi8 " ", *(int8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRIi16 " ", *(int16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRIi32 " ", *(int32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f ", *(float *)pVal->val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf ", *(double *)pVal->val); + break; + case TSDB_DATA_TYPE_VARCHAR: + printf("VARCHAR "); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_NCHAR: + printf("NCHAR "); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%" PRIu8 " ", *(uint8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%" PRIu16 " ", *(uint16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%" PRIu32 " ", *(uint32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%" PRIu64 " ", *(uint64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_JSON: + printf("JSON "); + break; + case TSDB_DATA_TYPE_VARBINARY: + printf("VARBIN "); + break; + case TSDB_DATA_TYPE_DECIMAL: + printf("DECIMAL "); + break; + case TSDB_DATA_TYPE_BLOB: + printf("BLOB "); + break; + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("MedBLOB "); + break; + // case TSDB_DATA_TYPE_BINARY: + // printf("BINARY "); + // break; + case TSDB_DATA_TYPE_MAX: + printf("UNDEF "); + break; + default: + printf("UNDEF "); + break; + } +} + +int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { + ASSERT(rows > 0); + int32_t result = 0; + + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + result += pDataCol->dataOff[rows - 1]; + SCellVal val = {0}; + if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { + TASSERT(0); + } + + // Currently, count the varDataTLen in of Null/None cols considering back compatibility test for 2.4 + result += varDataTLen(val.val); + // TODO: later on, don't save Null/None for VarDataT for 3.0 + // if (tdValTypeIsNorm(val.valType)) { + // result += varDataTLen(val.val); + // } + } else { + result += TYPE_BYTES[pDataCol->type] * rows; + } + + ASSERT(pDataCol->len == result); + + return result; +} + +bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); + tdGetKvRowValOfCol(pVal, pRow, pBitmap, offset, colIdx); + return true; +} + +bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, + SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); + return true; + } + void *pBitmap = tdGetBitmapAddrTp(pRow, flen); + tdGetTpRowValOfCol(pVal, pRow, pBitmap, colType, offset - sizeof(TSKEY), colIdx); + return true; +} + +int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { + if (isAllRowsNone(pCol)) { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + return TSDB_CODE_SUCCESS; + } + + if (TD_COL_ROWS_NORM(pCol)) { + pVal->valType = TD_VTYPE_NORM; + } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { + return terrno; + } + + if (tdValTypeIsNorm(pVal->valType)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pVal->val = POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + pVal->val = POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + } + } else { + pVal->valType = TD_VTYPE_NULL; +#ifdef TD_SUPPORT_READ2 + pVal->val = (void *)getNullValue(pCol->type); +#else + pVal->val = NULL; +#endif + } + return TSDB_CODE_SUCCESS; +} + +bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + if (TD_IS_TP_ROW(pIter->pRow)) { + STColumn *pCol = NULL; + STSchema *pSchema = pIter->pSchema; + while (pIter->colIdx < pSchema->numOfCols) { + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key + if (colId == pCol->colId) { + break; + } else if (pCol->colId < colId) { + ++pIter->colIdx; + continue; + } else { + return false; + } + } + tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; + } else if (TD_IS_KV_ROW(pIter->pRow)) { + return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); + } else { + pVal->valType = TD_VTYPE_NONE; + terrno = TSDB_CODE_INVALID_PARA; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + } + return true; +} + +bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + SKvRowIdx *pKvIdx = NULL; + bool colFound = false; + col_id_t kvNCols = tdRowGetNCols(pRow) - 1; + while (*nIdx < kvNCols) { + pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); + if (pKvIdx->colId == colId) { + ++(*nIdx); + pVal->val = POINTER_SHIFT(pRow, pKvIdx->offset); + colFound = true; + break; + } else if (pKvIdx->colId > colId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + ++(*nIdx); + } + } + + if (!colFound) { + if (colId <= pIter->maxColId) { + pVal->valType = TD_VTYPE_NONE; + return true; + } else { + return false; + } + } + +#ifdef TD_SUPPORT_BITMAP + int16_t colIdx = -1; + if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); + if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, SCellVal *pVal) { + STSRow *pRow = pIter->pRow; + if (IS_VAR_DATA_TYPE(colType)) { + pVal->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { + pVal->valType = TD_VTYPE_NONE; + } +#else + pVal->valType = isNull(pVal->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + + return true; +} + +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { + if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { + return 1; + } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { + return -1; + } else { + return 0; + } +} + +bool tdSTSRowGetVal(STSRowIter *pIter, col_id_t colId, col_type_t colType, SCellVal *pVal) { + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pVal->val = &pIter->pRow->ts; + pVal->valType = TD_VTYPE_NORM; + return true; + } + + STSRow *pRow = pIter->pRow; + int16_t colIdx = -1; + if (TD_IS_TP_ROW(pRow)) { + STSchema *pSchema = pIter->pSchema; + STColumn *pCol = + (STColumn *)taosbsearch(&colId, pSchema->columns, pSchema->numOfCols, sizeof(STColumn), tdCompareColId, TD_EQ); + if (!pCol) { + pVal->valType = TD_VTYPE_NONE; + if (COL_REACH_END(colId, pIter->maxColId)) return false; + return true; + } +#ifdef TD_SUPPORT_BITMAP + colIdx = POINTER_DISTANCE(pCol, pSchema->columns) / sizeof(STColumn); +#endif + tdGetTpRowValOfCol(pVal, pRow, pIter->pBitmap, pCol->type, pCol->offset - sizeof(TSKEY), colIdx - 1); + } else if (TD_IS_KV_ROW(pRow)) { + SKvRowIdx *pIdx = (SKvRowIdx *)taosbsearch(&colId, TD_ROW_COL_IDX(pRow), tdRowGetNCols(pRow), sizeof(SKvRowIdx), + compareKvRowColId, TD_EQ); +#ifdef TD_SUPPORT_BITMAP + if (pIdx) { + colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pIdx) / sizeof(SKvRowIdx); + } +#endif + tdGetKvRowValOfCol(pVal, pRow, pIter->pBitmap, pIdx ? pIdx->offset : -1, colIdx); + } else { + if (COL_REACH_END(colId, pIter->maxColId)) return false; + pVal->valType = TD_VTYPE_NONE; + } + + return true; +} + +static int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; + STColumn *pCol = (STColumn *)arg2; + + if (colId < pCol->colId) { + return -1; + } else if (colId == pCol->colId) { + return 0; + } else { + return 1; + } +} + +int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0xC0) >> 6); + break; + case 1: + *pValType = (((*pDestByte) & 0x30) >> 4); + break; + case 2: + *pValType = (((*pDestByte) & 0x0C) >> 2); + break; + case 3: + *pValType = ((*pDestByte) & 0x03); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0x80) >> 7); + break; + case 1: + *pValType = (((*pDestByte) & 0x40) >> 6); + break; + case 2: + *pValType = (((*pDestByte) & 0x20) >> 5); + break; + case 3: + *pValType = (((*pDestByte) & 0x10) >> 4); + break; + case 4: + *pValType = (((*pDestByte) & 0x08) >> 3); + break; + case 5: + *pValType = (((*pDestByte) & 0x04) >> 2); + break; + case 6: + *pValType = (((*pDestByte) & 0x02) >> 1); + break; + case 7: + *pValType = ((*pDestByte) & 0x01); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 7); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); + // *pDestByte |= (valType << 6); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); + // *pDestByte |= (valType << 5); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 4: + *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); + // *pDestByte |= (valType << 3); + break; + case 5: + *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 6: + *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); + // *pDestByte |= (valType << 1); + break; + case 7: + *pDestByte = ((*pDestByte) & 0xFE) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int32_t offset, int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + TASSERT(colIdx < tdRowGetNCols(pRow) - 1); + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + } +#else + TASSERT(0); + if (offset < 0) { + terrno = TSDB_CODE_INVALID_PARA; + output->valType = TD_VTYPE_NONE; + return terrno; + } + output->val = POINTER_SHIFT(pRow, offset); + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, + int16_t colIdx) { +#ifdef TD_SUPPORT_BITMAP + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { + output->valType = TD_VTYPE_NONE; + return terrno; + } + if (tdValTypeIsNorm(output->valType)) { + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + } +#else + if (IS_VAR_DATA_TYPE(colType)) { + output->val = POINTER_SHIFT(pRow, *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(pRow), offset)); + } else { + output->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); + } + output->valType = isNull(output->val, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t colId, int8_t colType, TDRowValT valType, const void *val, + bool isCopyVarData, int32_t offset, col_id_t colIdx) { + STSRow *pRow = pBuilder->pBuf; + if (!val) { +#ifdef TD_SUPPORT_BITMAP + if (tdValTypeIsNorm(valType)) { + terrno = TSDB_CODE_INVALID_PTR; + return terrno; + } +#else + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; +#endif + } + // TS KEY is stored in STSRow.ts and not included in STSRow.data field. + if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + TD_ROW_KEY(pRow) = *(TSKEY *)val; + // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. + return TSDB_CODE_SUCCESS; + } + // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. + if (TD_IS_TP_ROW(pRow)) { + tdAppendColValToTpRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset); + } else { + tdAppendColValToKvRow(pBuilder, valType, val, isCopyVarData, colType, colIdx, offset, colId); + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId) { + if ((offset < (int32_t)sizeof(SKvRowIdx)) || (colIdx < 1)) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(SKvRowIdx); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + // No need to store None/Null values. + if (tdValIsNorm(valType, val, colType)) { + // ts key stored in STSRow.ts + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(ptr, val, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + pColIdx->colId = colId; + pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN + const void *nullVal = getNullValue(colType); + + if (IS_VAR_DATA_TYPE(colType)) { + if (isCopyVarData) { + memcpy(ptr, nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(ptr, nullVal, TYPE_BYTES[colType]); + TD_ROW_LEN(row) += TYPE_BYTES[colType]; + } + } +#endif + + return 0; +} + +int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, + int8_t colType, int16_t colIdx, int32_t offset) { + if ((offset < (int32_t)sizeof(TSKEY)) || (colIdx < 1)) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + offset -= sizeof(TSKEY); + --colIdx; + +#ifdef TD_SUPPORT_BITMAP + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { + return terrno; + } +#endif + + STSRow *row = pBuilder->pBuf; + + // 1. No need to set flen part for Null/None, just use bitmap. When upsert for the same primary TS key, the bitmap + // should be updated simultaneously if Norm val overwrite Null/None cols. + // 2. When consume STSRow in memory by taos client/tq, the output of Null/None cols should both be Null. + if (tdValIsNorm(valType, val, colType)) { + // TODO: The layout of new data types imported since 3.0 like blob/medium blob is the same with binary/nchar. + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), val, varDataTLen(val)); + } + TD_ROW_LEN(row) += varDataTLen(val); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), val, TYPE_BYTES[colType]); + } + } +#ifdef TD_SUPPORT_BACK2 + // NULL/None value + else { + // TODO: Null value for new data types imported since 3.0 need to be defined. + const void *nullVal = getNullValue(colType); + if (IS_VAR_DATA_TYPE(colType)) { + // ts key stored in STSRow.ts + *(VarDataOffsetT *)POINTER_SHIFT(TD_ROW_DATA(row), offset) = TD_ROW_LEN(row); + + if (isCopyVarData) { + memcpy(POINTER_SHIFT(row, TD_ROW_LEN(row)), nullVal, varDataTLen(nullVal)); + } + TD_ROW_LEN(row) += varDataTLen(nullVal); + } else { + memcpy(POINTER_SHIFT(TD_ROW_DATA(row), offset), nullVal, TYPE_BYTES[colType]); + } + } +#endif + + return 0; +} + +int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen, + int32_t allNullLen, int32_t boundNullLen) { + if ((boundNullLen > 0) && (allNullLen > 0) && (nBoundCols > 0)) { + uint32_t tpLen = allNullLen; + uint32_t kvLen = sizeof(col_id_t) + sizeof(SKvRowIdx) * nBoundCols + boundNullLen; + if (isSelectKVRow(kvLen, tpLen)) { + pBuilder->rowType = TD_ROW_KV; + } else { + pBuilder->rowType = TD_ROW_TP; + } + + } else { + pBuilder->rowType = TD_ROW_TP; + } + + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TD_ROW_SET_INFO(pBuilder->pBuf, 0); + TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); +#endif + // the primary TS key is stored separatedly + len = TD_ROW_HEAD_LEN + pBuilder->flen - sizeof(TSKEY) + pBuilder->nBitmaps; + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); + memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps); +#endif + len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + + pBuilder->nBoundBitmaps; // add + TD_ROW_SET_LEN(pBuilder->pBuf, len); + TD_ROW_SET_SVER(pBuilder->pBuf, pBuilder->sver); + TD_ROW_SET_NCOLS(pBuilder->pBuf, pBuilder->nBoundCols); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { + pBuilder->pBuf = (STSRow *)pBuf; + if (!pBuilder->pBuf) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + TASSERT(pBuilder->nBitmaps > 0 && pBuilder->flen > 0); + + uint32_t len = 0; + switch (pBuilder->rowType) { + case TD_ROW_TP: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrTp(pBuilder->pBuf, pBuilder->flen); +#endif + break; + case TD_ROW_KV: +#ifdef TD_SUPPORT_BITMAP + pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); +#endif + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowInitEx(SRowBuilder *pBuilder, void *pBuf, uint32_t allNullLen, uint32_t boundNullLen, int32_t nCols, + int32_t nBoundCols, int32_t flen) { + if (tdSRowSetExtendedInfo(pBuilder, allNullLen, boundNullLen, nCols, nBoundCols, flen) < 0) { + return terrno; + } + return tdSRowResetBuf(pBuilder, pBuf); +} + +void tdSRowReset(SRowBuilder *pBuilder) { + pBuilder->rowType = TD_ROW_TP; + pBuilder->pBuf = NULL; + pBuilder->nBoundCols = -1; + pBuilder->nCols = -1; + pBuilder->flen = -1; + pBuilder->pBitmap = NULL; +} + +int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen) { + pBuilder->flen = flen; + pBuilder->nCols = nCols; + pBuilder->nBoundCols = nBoundCols; + if (pBuilder->flen <= 0 || pBuilder->nCols <= 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } +#ifdef TD_SUPPORT_BITMAP + // the primary TS key is stored separatedly + pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + if (nBoundCols > 0) { + pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + } else { + pBuilder->nBoundBitmaps = 0; + } +#else + pBuilder->nBitmaps = 0; + pBuilder->nBoundBitmaps = 0; +#endif + return TSDB_CODE_SUCCESS; +} + +int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdGetBitmapValTypeII(pBitmap, colIdx, pValType); + break; + case -1: + case 1: + tdGetBitmapValTypeI(pBitmap, colIdx, pValType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { + TDRowValT valType = 0; + tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); + if (tdValTypeIsNorm(valType)) { + return true; + } + return false; +} + +int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS; + int16_t nOffset = colIdx & TD_VTYPE_OPTR; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + // set the value and clear other partitions for offset 0 + // *pDestByte |= (valType << 6); + break; + case 1: + *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + // *pDestByte |= (valType << 4); + break; + case 2: + *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + // *pDestByte |= (valType << 2); + break; + case 3: + *pDestByte = ((*pDestByte) & 0xFC) | valType; + // *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdSetBitmapValTypeII(pBitmap, colIdx, valType); + break; + case -1: + case 1: + tdSetBitmapValTypeI(pBitmap, colIdx, valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_t flen, col_id_t nKvCols) { +#ifdef TD_SUPPORT_BITMAP + switch (rowType) { + case TD_ROW_TP: + return tdGetBitmapAddrTp(pRow, flen); + case TD_ROW_KV: + return tdGetBitmapAddrKv(pRow, nKvCols); + default: + break; + } +#endif + return NULL; +} + +void tdSTSRowIterReset(STSRowIter *pIter, STSRow *pRow) { + pIter->pRow = pRow; + pIter->pBitmap = tdGetBitmapAddr(pRow, pRow->type, pIter->pSchema->flen, tdRowGetNCols(pRow)); + pIter->offset = 0; + pIter->colIdx = PRIMARYKEY_TIMESTAMP_COL_ID; + pIter->kvIdx = 0; +} + +void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) { + pIter->pSchema = pSchema; + pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; } \ No newline at end of file -- GitLab