diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3ce6c1684bf5c466a3f886d7602ba625f46d8b54..95694353d3948951f06e400123943b135a9890ab 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -269,6 +269,8 @@ typedef struct SSchema { #define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_NAME(s) ((s)->name) +STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; diff --git a/include/common/trow.h b/include/common/trow.h index 40462a7eefefd498f53df1f6ad97361743914393..a98b03fd5372ad65ab418b2998060cb2427e2def 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -219,7 +219,7 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return #define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow? -void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap); +void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); @@ -308,8 +308,8 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: + *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); // set the value and clear other partitions for offset 0 - *pDestByte = (valType << 6); // *pDestByte |= (valType << 6); break; case 1: @@ -417,8 +417,8 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: + *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); // set the value and clear other partitions for offset 0 - *pDestByte = (valType << 7); // *pDestByte |= (valType << 7); break; case 1: @@ -1107,11 +1107,11 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col if (TD_IS_TP_ROW(pIter->pRow)) { STColumn *pCol = NULL; STSchema *pSchema = pIter->pSchema; - while (pIter->colIdx <= pSchema->numOfCols) { + while (pIter->colIdx < pSchema->numOfCols) { pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key if (colId == pCol->colId) { break; - } else if (colId < pCol->colId) { + } else if (pCol->colId < colId) { ++pIter->colIdx; continue; } else { @@ -1237,6 +1237,101 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, return result; } +static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { + if (tdValTypeIsNull(pVal->valType)) { + printf("NULL "); + return; + } else if (tdValTypeIsNone(pVal->valType)) { + printf("NONE "); + return; + } + switch (colType) { + case TSDB_DATA_TYPE_BOOL: + printf("%s ", (*(int8_t *)pVal->val) == 0 ? "false" : "true"); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%" PRIi8 " ", *(int8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%" PRIi16 " ", *(int16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_INT: + printf("%" PRIi32 " ", *(int32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_FLOAT: + printf("%f ", *(float *)pVal->val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%lf ", *(double *)pVal->val); + break; + case TSDB_DATA_TYPE_VARCHAR: + printf("VARCHAR "); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%" PRIi64 " ", *(int64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_NCHAR: + printf("NCHAR "); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%" PRIu8 " ", *(uint8_t *)pVal->val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%" PRIu16 " ", *(uint16_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%" PRIu32 " ", *(uint32_t *)pVal->val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%" PRIu64 " ", *(uint64_t *)pVal->val); + break; + case TSDB_DATA_TYPE_JSON: + printf("JSON "); + break; + case TSDB_DATA_TYPE_VARBINARY: + printf("VARBIN "); + break; + case TSDB_DATA_TYPE_DECIMAL: + printf("DECIMAL "); + break; + case TSDB_DATA_TYPE_BLOB: + printf("BLOB "); + break; + case TSDB_DATA_TYPE_MEDIUMBLOB: + printf("MedBLOB "); + break; + // case TSDB_DATA_TYPE_BINARY: + // printf("BINARY "); + // break; + case TSDB_DATA_TYPE_MAX: + printf("UNDEF "); + break; + default: + printf("UNDEF "); + break; + } +} + +static void tdSRowPrint(STSRow *row, STSchema *pSchema) { + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pSchema); + tdSTSRowIterReset(&iter, row); + printf(">>>"); + for (int i = 0; i < pSchema->numOfCols; ++i) { + STColumn *stCol = pSchema->columns + i; + SCellVal sVal = {.valType = 255, .val = NULL}; + if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { + break; + } + ASSERT(sVal.valType == 0 || sVal.valType == 1 || sVal.valType == 2); + tdSCellValPrint(&sVal, stCol->type); + } + printf("\n"); +} + #ifdef TROW_ORIGIN_HZ typedef struct { uint32_t nRows; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 4fb7531dc7ab59db3d50aff0f84fb1392ff95e4f..9090ebebdb4512b882f3fec03861cdcb5ef5f352 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3646,3 +3646,27 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); } + +STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { + STSchemaBuilder schemaBuilder = {0}; + if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) { + return NULL; + } + + for (int i = 0; i < nCols; i++) { + SSchema *schema = *pSchema + i; + if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { + tdDestroyTSchemaBuilder(&schemaBuilder); + return NULL; + } + } + + STSchema *pNSchema = tdGetSchemaFromBuilder(&schemaBuilder); + if (pNSchema == NULL) { + tdDestroyTSchemaBuilder(&schemaBuilder); + return NULL; + } + + tdDestroyTSchemaBuilder(&schemaBuilder); + return pNSchema; +} diff --git a/source/common/src/trow.c b/source/common/src/trow.c index a1a2d236f90b58c9b1bb0122af22e21ab98d35a3..ba05d732c3210c1f6d0eb1d52b87c55d7d30b50e 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -24,9 +24,8 @@ const uint8_t tdVTypeByte[2][3] = {{ }, { // 1 bit - TD_VTYPE_NORM_BYTE_I, - TD_VTYPE_NULL_BYTE_I, - TD_VTYPE_NULL_BYTE_I, // padding + TD_VTYPE_NORM_BYTE_I, TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NULL_BYTE_I, // padding } }; @@ -224,17 +223,40 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) { * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker. * * @param srcBitmap - * @param srcLen + * @param nBits * @param dstBitmap */ -void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap) { +void tdMergeBitmap(uint8_t *srcBitmap, int32_t nBits, uint8_t *dstBitmap) { int32_t i = 0, j = 0; + int32_t nBytes = TD_BITMAP_BYTES(nBits); + int32_t nStrictBytes = nBits / 4; + int32_t nPartialBits = nBits - nStrictBytes * 4; - if (srcLen > 0) { + switch (nPartialBits) { + case 0: + // NOTHING TODO + break; + case 1: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xC0; + } break; + case 2: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xF0; + } break; + case 3: { + void *lastByte = POINTER_SHIFT(srcBitmap, nStrictBytes); + *(uint8_t *)lastByte &= 0xFC; + } break; + default: + ASSERT(0); + } + + if (nBytes > 0) { dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); } - while ((++i) < srcLen) { + while ((++i) < nBytes) { if ((i & 1) == 0) { dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); } else { @@ -381,17 +403,18 @@ STSRow *tdRowDup(STSRow *row) { } /** - * @brief - * - * @param pCol - * @param valType - * @param val - * @param numOfRows - * @param maxPoints + * @brief + * + * @param pCol + * @param valType + * @param val + * @param numOfRows + * @param maxPoints * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) - * @return int + * @return int */ -int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, int8_t bitmapMode) { +int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, + int8_t bitmapMode) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -426,7 +449,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int pCol->len += pCol->bytes; } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); #endif return 0; } @@ -537,7 +560,8 @@ int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCol return TSDB_CODE_SUCCESS; } -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, TDRowVerT maxVer) { +int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull, + TDRowVerT maxVer) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfCols == source->numOfCols); int offset = 0; @@ -558,7 +582,8 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } } ++target->numOfRows; @@ -605,7 +630,8 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } } @@ -621,12 +647,14 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i TASSERT(0); } if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, + target->bitmapMode); } else if (target->cols[i].len > 0) { dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 09616a8969583abae05e152511ce886983a62823..28ba50c3df6f4451d74fdb9d985760d5ed4168e9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1350,7 +1350,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF if (tBitmaps > 0) { bptr = POINTER_SHIFT(pBlockData, lsize + flen); if (isSuper && !tdDataColsIsBitmapI(pDataCols)) { - tdMergeBitmap((uint8_t *)pDataCol->pBitmap, nBitmaps, (uint8_t *)pDataCol->pBitmap); + tdMergeBitmap((uint8_t *)pDataCol->pBitmap, rowsToWrite, (uint8_t *)pDataCol->pBitmap); } tBitmapsLen = tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index b15271a51a0d2c5b0229b19ac49f06f0f4478bd4..0cf8a0d359926512e1ebe55c2db4075543ed6abd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -305,7 +305,7 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; if (pDataCol->bitmap) { ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); - tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap); + tdMergeBitmap(pDataCol->pBitmap, pReadh->pDCols[0]->numOfRows, pDataCol->pBitmap); tdDataColsSetBitmapI(pReadh->pDCols[0]); } }