diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 991cb55e5070a55470278554a73e4f49c3e32184..8e07d4e6699a87142568d8073ec3801caf7e3fde 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -131,12 +131,17 @@ typedef struct { STColumn *columns; } STSchemaBuilder; -#define TD_VTYPE_BITS 2 // val type -#define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 -#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder - -#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS)) -#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8)) +// use 2 bits for bitmap(default: STSRow/sub block) +#define TD_VTYPE_BITS 2 +#define TD_VTYPE_PARTS 4 // PARTITIONS: 1 byte / 2 bits +#define TD_VTYPE_OPTR 3 // OPERATOR: 4 - 1, utilize to get remainder +#define TD_BITMAP_BYTES(cnt) (((cnt) + TD_VTYPE_OPTR) >> 2) + +// use 1 bit for bitmap(super block) +#define TD_VTYPE_BITS_I 1 +#define TD_VTYPE_PARTS_I 8 // PARTITIONS: 1 byte / 1 bit +#define TD_VTYPE_OPTR_I 7 // OPERATOR: 8 - 1, utilize to get remainder +#define TD_BITMAP_BYTES_I(cnt) (((cnt) + TD_VTYPE_OPTR_I) >> 3) int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); @@ -367,9 +372,10 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche // ----------------- Data column structure // SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { - int8_t type; // column type - uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint8_t reserve : 7; + int8_t type; // column type + uint8_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows + uint8_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit) + uint8_t reserve : 6; int16_t colId; // column ID int32_t bytes; // column data bytes defined int32_t offset; // data offset in a SDataRow (including the header size) @@ -381,6 +387,8 @@ typedef struct SDataCol { TSKEY ts; // only used in last NULL column } SDataCol; + + #define isAllRowsNull(pCol) ((pCol)->len == 0) #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } @@ -421,10 +429,14 @@ typedef struct { col_id_t numOfCols; // Total number of cols int32_t maxPoints; // max number of points int32_t numOfRows; - int32_t sversion; // TODO: set sversion + int32_t bitmapMode : 1; // default is 0(2 bits), otherwise 1(1 bit) + int32_t sversion : 31; // TODO: set sversion(not used yet) SDataCol *cols; } SDataCols; +static FORCE_INLINE bool tdDataColsIsBitmapI(SDataCols *pCols) { return pCols->bitmapMode != 0; } +static FORCE_INLINE void tdDataColsSetBitmapI(SDataCols *pCols) { pCols->bitmapMode = 1; } + #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) diff --git a/include/common/trow.h b/include/common/trow.h index 7cde8c50c5825766d78c2498a55db79e162dcbc6..b7ffcd14c642d07d88e8bed20f2a7ecd394c9895 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -40,21 +40,20 @@ extern "C" { #define TD_ROW_KV 0x01U /** - * @brief val type - * - for data from client input and STSRow in memory, 3 types of val none/null/norm available - * - for data in + * @brief value type + * - for data from client input and STSRow in memory, 3 types of value none/null/norm available */ -#define TD_VTYPE_NONE 0x0U // none or unknown/undefined +#define TD_VTYPE_NORM 0x00U // normal val: not none, not null(no need assign value) #define TD_VTYPE_NULL 0x01U // null val -#define TD_VTYPE_NORM 0x02U // normal val: not none, not null -#define TD_VTYPE_MAX 0x03U // +#define TD_VTYPE_NONE 0x02U // none or unknown/undefined +#define TD_VTYPE_MAX 0x03U // -#define TD_VTYPE_NONE_BYTE 0x0U +#define TD_VTYPE_NORM_BYTE 0x0U #define TD_VTYPE_NULL_BYTE 0x55U -#define TD_VTYPE_NORM_BYTE 0xAAU +#define TD_VTYPE_NONE_BYTE 0xAAU -#define TD_ROWS_ALL_NORM 0x01U -#define TD_ROWS_NULL_NORM 0x0U +#define TD_ROWS_ALL_NORM 0x00U +#define TD_ROWS_NULL_NORM 0x01U #define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) @@ -126,15 +125,15 @@ typedef struct { uint32_t info; struct { /// row type - uint32_t type : 2; + uint16_t type : 2; /// is delete row(0 not delete, 1 delete) - uint32_t del : 1; + uint16_t del : 1; /// endian(0 little endian, 1 big endian) - uint32_t endian : 1; + uint16_t endian : 1; /// reserved for back compatibility - uint32_t reserve : 12; + uint16_t reserve : 12; /// row schema version - uint32_t sver : 16; + uint16_t sver; }; }; /// row total length @@ -216,11 +215,16 @@ static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return #define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow? +void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap); static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, TD_ROW_LEN(row)); } -static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType); -int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType); -static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType); -int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints); +static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType); +static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); +static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); +int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); +static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType); +static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType); +int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, + int8_t bitmapMode); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -259,15 +263,32 @@ static FORCE_INLINE void *tdGetBitmapAddr(STSRow *pRow, uint8_t rowType, uint32_ return NULL; } +static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdSetBitmapValTypeII(pBitmap, colIdx, valType); + break; + case -1: + case 1: + tdSetBitmapValTypeI(pBitmap, colIdx, valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + /** - * @brief + * @brief Use 2 bits at default * * @param pBitmap * @param colIdx The relative index of colId, may have minus value as parameter. * @param valType * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType) { +static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType) { if (!pBitmap || colIdx < 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; @@ -276,18 +297,24 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: - *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + // *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); + // set the value and clear other partitions for offset 0 + *pDestByte = (valType << 6); break; case 1: - *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + // *pDestByte = ((*pDestByte) & 0xCF) | (valType << 4); + *pDestByte |= (valType << 4); break; case 2: - *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + // *pDestByte = ((*pDestByte) & 0xF3) | (valType << 2); + *pDestByte |= (valType << 2); break; case 3: - *pDestByte = ((*pDestByte) & 0xFC) | valType; + // *pDestByte = ((*pDestByte) & 0xFC) | valType; + *pDestByte |= (valType); break; default: TASSERT(0); @@ -297,15 +324,32 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD return TSDB_CODE_SUCCESS; } +static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { + switch (bitmapMode) { + case 0: + tdGetBitmapValTypeII(pBitmap, colIdx, pValType); + break; + case -1: + case 1: + tdGetBitmapValTypeI(pBitmap, colIdx, pValType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + /** - * @brief + * @brief Use 2 bits at default * * @param pBitmap * @param colIdx The relative index of colId, may have minus value as parameter. * @param pValType * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType) { +static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; @@ -314,6 +358,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -335,6 +380,117 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD return TSDB_CODE_SUCCESS; } +/** + * @brief + * + * @param pBitmap + * @param colIdx The relative index of colId, may have minus value as parameter. + * @param valType + * @return FORCE_INLINE + */ +static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT valType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + // *pDestByte = ((*pDestByte) & 0x7F) | (valType << 7); + // set the value and clear other partitions for offset 0 + *pDestByte = (valType << 7); + break; + case 1: + // *pDestByte = ((*pDestByte) & 0xBF) | (valType << 6); + *pDestByte |= (valType << 6); + break; + case 2: + // *pDestByte = ((*pDestByte) & 0xDF) | (valType << 5); + *pDestByte |= (valType << 5); + break; + case 3: + // *pDestByte = ((*pDestByte) & 0xEF) | (valType << 4); + *pDestByte |= (valType << 4); + break; + case 4: + // *pDestByte = ((*pDestByte) & 0xF7) | (valType << 3); + *pDestByte |= (valType << 3); + break; + case 5: + // *pDestByte = ((*pDestByte) & 0xFB) | (valType << 2); + *pDestByte |= (valType << 2); + break; + case 6: + // *pDestByte = ((*pDestByte) & 0xFD) | (valType << 1); + *pDestByte |= (valType << 1); + break; + case 7: + // *pDestByte = ((*pDestByte) & 0xFE) | valType; + *pDestByte |= (valType); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pBitmap + * @param colIdx The relative index of colId, may have minus value as parameter. + * @param pValType + * @return FORCE_INLINE + */ +static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType) { + if (!pBitmap || colIdx < 0) { + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + int16_t nBytes = colIdx / TD_VTYPE_PARTS_I; + int16_t nOffset = colIdx & TD_VTYPE_OPTR_I; + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + // use literal value directly and not use formula to simplify the codes + switch (nOffset) { + case 0: + *pValType = (((*pDestByte) & 0x80) >> 7); + break; + case 1: + *pValType = (((*pDestByte) & 0x40) >> 6); + break; + case 2: + *pValType = (((*pDestByte) & 0x20) >> 5); + break; + case 3: + *pValType = (((*pDestByte) & 0x10) >> 4); + break; + case 4: + *pValType = (((*pDestByte) & 0x08) >> 3); + break; + case 5: + *pValType = (((*pDestByte) & 0x04) >> 2); + break; + case 6: + *pValType = (((*pDestByte) & 0x02) >> 1); + break; + case 7: + *pValType = ((*pDestByte) & 0x01); + break; + default: + TASSERT(0); + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + return TSDB_CODE_SUCCESS; +} + // ----------------- Tuple row structure(STpRow) /* * |<----------------------------- tlen ---------------------------------->| @@ -552,7 +708,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa --colIdx; #ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) { + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { return terrno; } #endif @@ -610,7 +766,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa --colIdx; #ifdef TD_SUPPORT_BITMAP - if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) { + if (tdSetBitmapValType(pBuilder->pBitmap, colIdx, valType, 0) != TSDB_CODE_SUCCESS) { return terrno; } #endif @@ -691,12 +847,6 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { TD_ROW_KEY(pRow) = *(TSKEY *)val; // The primary TS key is Norm all the time, thus its valType is not stored in bitmap. - // #ifdef TD_SUPPORT_BITMAP - // pBitmap = tdGetBitmapAddr(pRow, pRow->type, pBuilder->flen, pRow->ncols); - // if (tdSetBitmapValType(pBitmap, colIdx, valType) != TSDB_CODE_SUCCESS) { - // return terrno; - // } - // #endif return TSDB_CODE_SUCCESS; } // TODO: We can avoid the type judegement by FP, but would prevent the inline scheme. @@ -712,7 +862,7 @@ static FORCE_INLINE int32_t tdAppendColValToRow(SRowBuilder *pBuilder, col_id_t static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, void *pBitmap, int8_t colType, int32_t offset, int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; } @@ -748,7 +898,7 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v int16_t colIdx) { #ifdef TD_SUPPORT_BITMAP TASSERT(colIdx < tdRowGetNCols(pRow) - 1); - if (tdGetBitmapValType(pBitmap, colIdx, &output->valType) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pBitmap, colIdx, &output->valType, 0) != TSDB_CODE_SUCCESS) { output->valType = TD_VTYPE_NONE; return terrno; } @@ -868,7 +1018,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy } #ifdef TD_SUPPORT_BITMAP - if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } #else @@ -905,7 +1055,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, #ifdef TD_SUPPORT_BITMAP int16_t colIdx = -1; if (pKvIdx) colIdx = POINTER_DISTANCE(TD_ROW_COL_IDX(pRow), pKvIdx) / sizeof(SKvRowIdx); - if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType) != TSDB_CODE_SUCCESS) { + if (tdGetBitmapValType(pIter->pBitmap, colIdx, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; } #else @@ -962,7 +1112,7 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); // Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row) { +static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row, int8_t bitmapMode) { if (isAllRowsNone(pCol)) { pVal->valType = TD_VTYPE_NULL; #ifdef TD_SUPPORT_READ2 @@ -975,7 +1125,7 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in if (TD_COL_ROWS_NORM(pCol)) { pVal->valType = TD_VTYPE_NORM; - } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType)) < 0) { + } else if (tdGetBitmapValType(pCol->pBitmap, row, &(pVal->valType), bitmapMode) < 0) { return terrno; } @@ -1018,14 +1168,14 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o return true; } -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { +static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows, int8_t bitmapMode) { ASSERT(rows > 0); int32_t result = 0; if (IS_VAR_DATA_TYPE(pDataCol->type)) { result += pDataCol->dataOff[rows - 1]; SCellVal val = {0}; - if (tdGetColDataOfRow(&val, pDataCol, rows - 1) < 0) { + if (tdGetColDataOfRow(&val, pDataCol, rows - 1, bitmapMode) < 0) { TASSERT(0); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index c1225012ac63baf3c489e0525504b6fca9e5b352..f6614633fb50a4ba60534e2a785755f809563beb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -266,6 +266,8 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } + +#if 0 // value from timestamp should be TKEY here instead of TSKEY int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); @@ -297,7 +299,7 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo } return 0; } - +#endif static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) { if (IS_VAR_DATA_TYPE(pCol->type)) { return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); @@ -314,6 +316,7 @@ bool isNEleNull(SDataCol *pCol, int nEle) { return true; } +#if 0 static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; @@ -326,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } } -static void dataColSetNEleNull(SDataCol *pCol, int nEle) { +static void dataColSetNEleNull(SDataCol *pCol, int nEle, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; for (int i = 0; i < nEle; ++i) { @@ -337,7 +340,7 @@ static void dataColSetNEleNull(SDataCol *pCol, int nEle) { pCol->len = TYPE_BYTES[pCol->type] * nEle; } } - +#endif void *dataColSetOffset(SDataCol *pCol, int nEle) { ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR))); @@ -364,6 +367,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->maxCols = maxCols; pCols->numOfRows = 0; pCols->numOfCols = 0; + // pCols->bitmapMode = 0; // calloc already set 0 if (maxCols > 0) { pCols->cols = (SDataCol *)taosMemoryCalloc(maxCols, sizeof(SDataCol)); @@ -472,6 +476,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; + pCols->bitmapMode = 0; for (int i = 0; i < pCols->maxCols; ++i) { dataColReset(pCols->cols + i); } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 0d5a874c1c8a0eaf1915a7afb25a96b154b91855..d9c48bfe7684ea54f0e1ceb9400f03beb2f36973 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -22,11 +22,224 @@ const uint8_t tdVTypeByte[3] = { TD_VTYPE_NULL_BYTE, // TD_VTYPE_NULL }; +// declaration +static uint8_t tdGetBitmapByte(uint8_t byte); + +// implementation +/** + * @brief Compress bitmap bytes comprised of 2-bits to counterpart of 1-bit. + * e.g. + * TD_VTYPE_NORM 0x00U(00000000) to 00000000 Normal + * TD_VTYPE_NULL 0x01U(00000001) to 00000001 Null + * TD_VTYPE_NONE 0x02U(00000010) to 00000001 Null + * + * 00000000 0x00 0x00 + * 01000000 0x40 0x08 + * 10000000 0x80 0x08 + * ... + * @param byte + * @return uint8_t + */ +static uint8_t tdGetMergedBitmapByte(uint8_t byte) { + switch (byte) { + case 0x00: + return 0x00; + case 0x40: + return 0x08; + case 0x80: + return 0x08; + case 0x10: + return 0x04; + case 0x50: + return 0x0c; + case 0x90: + return 0x0c; + case 0x20: + return 0x04; + case 0x60: + return 0x0c; + case 0xa0: + return 0x0c; + case 0x04: + return 0x02; + case 0x44: + return 0x0a; + case 0x84: + return 0x0a; + case 0x14: + return 0x06; + case 0x54: + return 0x0e; + case 0x94: + return 0x0e; + case 0x24: + return 0x06; + case 0x64: + return 0x0e; + case 0xa4: + return 0x0e; + case 0x08: + return 0x02; + case 0x48: + return 0x0a; + case 0x88: + return 0x0a; + case 0x18: + return 0x06; + case 0x58: + return 0x0e; + case 0x98: + return 0x0e; + case 0x28: + return 0x06; + case 0x68: + return 0x0e; + case 0xa8: + return 0x0e; + case 0x01: + return 0x01; + case 0x41: + return 0x09; + case 0x81: + return 0x09; + case 0x11: + return 0x05; + case 0x51: + return 0x0d; + case 0x91: + return 0x0d; + case 0x21: + return 0x05; + case 0x61: + return 0x0d; + case 0xa1: + return 0x0d; + case 0x05: + return 0x03; + case 0x45: + return 0x0b; + case 0x85: + return 0x0b; + case 0x15: + return 0x07; + case 0x55: + return 0x0f; + case 0x95: + return 0x0f; + case 0x25: + return 0x07; + case 0x65: + return 0x0f; + case 0xa5: + return 0x0f; + case 0x09: + return 0x03; + case 0x49: + return 0x0b; + case 0x89: + return 0x0b; + case 0x19: + return 0x07; + case 0x59: + return 0x0f; + case 0x99: + return 0x0f; + case 0x29: + return 0x07; + case 0x69: + return 0x0f; + case 0xa9: + return 0x0f; + case 0x02: + return 0x01; + case 0x42: + return 0x09; + case 0x82: + return 0x09; + case 0x12: + return 0x05; + case 0x52: + return 0x0d; + case 0x92: + return 0x0d; + case 0x22: + return 0x05; + case 0x62: + return 0x0d; + case 0xa2: + return 0x0d; + case 0x06: + return 0x03; + case 0x46: + return 0x0b; + case 0x86: + return 0x0b; + case 0x16: + return 0x07; + case 0x56: + return 0x0f; + case 0x96: + return 0x0f; + case 0x26: + return 0x07; + case 0x66: + return 0x0f; + case 0xa6: + return 0x0f; + case 0x0a: + return 0x03; + case 0x4a: + return 0x0b; + case 0x8a: + return 0x0b; + case 0x1a: + return 0x07; + case 0x5a: + return 0x0f; + case 0x9a: + return 0x0f; + case 0x2a: + return 0x07; + case 0x6a: + return 0x0f; + case 0xaa: + return 0x0f; + default: + // make sure the bitmap area is set to 0 firstly + ASSERT(0); + return 0x0f; // return NULL bitmap for exception + } +} + +/** + * @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker. + * + * @param srcBitmap + * @param srcLen + * @param dstBitmap + */ +void tdMergeBitmap(uint8_t *srcBitmap, int32_t srcLen, uint8_t *dstBitmap) { + int32_t i = 0, j = 0; + + if (srcLen > 0) { + dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); + } + + while ((++i) < srcLen) { + if ((i & 1) == 0) { + dstBitmap[j] = (tdGetMergedBitmapByte(srcBitmap[i]) << 4); + } else { + dstBitmap[j] |= tdGetMergedBitmapByte(srcBitmap[i]); + ++j; + } + } +} + // static void dataColSetNEleNull(SDataCol *pCol, int nEle); static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows, bool forceSetNull); -static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap) { +static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); @@ -37,7 +250,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit pCol->len += TYPE_BYTES[pCol->type]; } if (setBitmap) { - tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); + tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE, bitmapMode); } } @@ -53,7 +266,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit // } // } -int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType) { +int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) { TASSERT(valType < TD_VTYPE_MAX); int16_t nBytes = nEle / TD_VTYPE_PARTS; for (int i = 0; i < nBytes; ++i) { @@ -63,12 +276,12 @@ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType) { int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS; for (int j = 0; j < nLeft; ++j) { - tdSetBitmapValType(pBitmap, j, valType); + tdSetBitmapValType(pBitmap, j, valType, bitmapMode); } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap) { +static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; char *ptr = POINTER_SHIFT(pCol->pData, pCol->len); @@ -79,22 +292,22 @@ static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBit pCol->len += TYPE_BYTES[pCol->type]; } if (setBitmap) { - tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE); + tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE, bitmapMode); } } -static void dataColSetNEleNone(SDataCol *pCol, int nEle) { +static void dataColSetNEleNone(SDataCol *pCol, int nEle, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; for (int i = 0; i < nEle; ++i) { - dataColSetNoneAt(pCol, i, false); + dataColSetNoneAt(pCol, i, false, bitmapMode); } } else { setNullN(pCol->pData, pCol->type, pCol->bytes, nEle); pCol->len = TYPE_BYTES[pCol->type] * nEle; } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValTypeN(pCol->pBitmap, nEle, TD_VTYPE_NONE); + tdSetBitmapValTypeN(pCol->pBitmap, nEle, TD_VTYPE_NONE, bitmapMode); #endif } @@ -126,7 +339,18 @@ STSRow *tdRowDup(STSRow *row) { return trow; } -int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints) { +/** + * @brief + * + * @param pCol + * @param valType + * @param val + * @param numOfRows + * @param maxPoints + * @param bitmapMode default is 0(2 bits), otherwise 1(1 bit) + * @return int + */ +int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints, int8_t bitmapMode) { TASSERT(pCol != NULL); // Assume that the columns not specified during insert/upsert mean None. @@ -139,7 +363,7 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1; if (numOfRows > 0) { // Find the first not None value, fill all previous values as None - dataColSetNEleNone(pCol, numOfRows); + dataColSetNEleNone(pCol, numOfRows, bitmapMode); } } if (!tdValTypeIsNorm(valType)) { @@ -161,7 +385,8 @@ int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int pCol->len += pCol->bytes; } #ifdef TD_SUPPORT_BITMAP - tdSetBitmapValType(pCol->pBitmap, numOfRows, valType); + + tdSetBitmapValType(pCol->pBitmap, numOfRows, valType, bitmapMode); #endif return 0; } @@ -175,14 +400,13 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols void *pBitmap = tdGetBitmapAddrTp(pRow, pSchema->flen); SDataCol *pDataCol = &(pCols->cols[0]); - if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); - } + ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= schemaNCols(pSchema)) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; continue; } @@ -193,13 +417,13 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; ++rcol; } else if (pRowCol->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; } } @@ -218,14 +442,13 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow)); SDataCol *pDataCol = &(pCols->cols[0]); - if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints); - } + ASSERT(pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); while (dcol < pCols->numOfCols) { pDataCol = &(pCols->cols[dcol]); if (rcol >= tRowCols || rcol >= tSchemaCols) { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; continue; } @@ -240,13 +463,13 @@ static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) { return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; ++rcol; } else if (pIdx->colId < pDataCol->colId) { ++rcol; } else { - tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints); + tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode); ++dcol; } } @@ -291,10 +514,10 @@ int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int * for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0 || target->cols[j].len > 0) { SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset)) < 0) { + if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset), source->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints); + tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); } } ++target->numOfRows; @@ -338,10 +561,10 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0 || target->cols[i].len > 0) { SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) { + if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); } } @@ -353,18 +576,18 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i for (int i = 0; i < src2->numOfCols; i++) { SCellVal sVal = {0}; ASSERT(target->cols[i].type == src2->cols[i].type); - if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2) < 0) { + if (tdGetColDataOfRow(&sVal, src2->cols + i, *iter2, src2->bitmapMode) < 0) { TASSERT(0); } if (src2->cols[i].len > 0 && !tdValTypeIsNull(sVal.valType)) { - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { - if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) { + if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1, src1->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints); + tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints, target->bitmapMode); } else if (target->cols[i].len > 0) { - dataColSetNullAt(&target->cols[i], target->numOfRows, true); + dataColSetNullAt(&target->cols[i], target->numOfRows, true, target->bitmapMode); } } target->numOfRows++; @@ -480,6 +703,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (pRet == NULL) return NULL; pRet->numOfCols = pDataCols->numOfCols; + pRet->bitmapMode = pDataCols->bitmapMode; pRet->sversion = pDataCols->sversion; if (keepData) pRet->numOfRows = pDataCols->numOfRows; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 97a0832b9c893b496992ecfa60139be92f40129a..b85caad4c8bf97e29ea0c495c00ea4f5613480c3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -264,6 +264,9 @@ typedef struct { #define SBlock SBlockV0 // latest SBlock definition +static FORCE_INLINE bool tsdbIsSupBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 1; } +static FORCE_INLINE bool tsdbIsSubBlock(SBlock *pBlock) { return pBlock->numOfSubBlocks == 0; } + #endif typedef struct { @@ -276,7 +279,7 @@ typedef struct { #ifdef TD_REFACTOR_3 typedef struct { int16_t colId; - uint16_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint16_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows uint16_t reserve : 15; int32_t len; uint32_t type : 8; @@ -295,7 +298,7 @@ typedef struct { int16_t colId; uint16_t type : 6; uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) - uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t bitmap : 1; // 0: no bitmap if all rows are NORM, 1: has bitmap if has NULL/NORM rows uint32_t len : 31; // data length + bitmap length uint32_t offset; } SBlockColV0; @@ -404,7 +407,7 @@ int tsdbSetReadTable(SReadH *pReadh, STable *pTable); int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget); int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo); int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, - int numOfColsIds); + int numOfColsIds, bool mergeBitmap); int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock); int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx); void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 42d29209f5f07060c1fae50646c676f2b1b50235..21ad1ec076f7916ea99504b82252fc5da4ebbc58 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1280,7 +1280,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - // int32_t tBitmaps = 0; + int32_t sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps; for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish @@ -1292,25 +1292,13 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; int32_t flen; // final length - int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); + int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite, pDataCols->bitmapMode); #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; int32_t tBitmapsLen = 0; if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { - tBitmaps = nBitmaps; -#if 0 - if (IS_VAR_DATA_TYPE(pDataCol->type)) { - tBitmaps = nBitmaps; - tlen += tBitmaps; - } else { - tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); - tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; - } -#endif - // move bitmap parts ahead - // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + tBitmaps = sBitmaps; } #endif @@ -1340,6 +1328,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tlen + COMP_OVERFLOW_BYTES); if (tBitmaps > 0) { bptr = POINTER_SHIFT(pBlockData, lsize + flen); + if (isSuper && !tdDataColsIsBitmapI(pDataCols)) { + tdMergeBitmap((uint8_t *)pDataCol->pBitmap, nBitmaps, (uint8_t *)pDataCol->pBitmap); + } tBitmapsLen = tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); @@ -1506,7 +1497,7 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { } SSkipListIterator titer = *(pIter->pIter); - if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1; + if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1, false) < 0) return -1; tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo); @@ -1656,6 +1647,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey); tdResetDataCols(pTarget); + pTarget->bitmapMode = pDataCols->bitmapMode; + while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); STSRow *row = tsdbNextIterRow(pCommitIter->pIter); @@ -1668,17 +1661,17 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 == INT64_MAX && key2 == INT64_MAX) break; if (key1 < key2) { - for (int i = 0; i < pDataCols->numOfCols; i++) { + for (int i = 0; i < pDataCols->numOfCols; ++i) { // TODO: dataColAppendVal may fail SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { + if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); + tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode); } - pTarget->numOfRows++; - (*iter)++; + ++pTarget->numOfRows; + ++(*iter); } else if (key1 > key2) { if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) { pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row)); @@ -1691,13 +1684,13 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt } else { if (update != TD_ROW_OVERWRITE_UPDATE) { // copy disk data - for (int i = 0; i < pDataCols->numOfCols; i++) { + for (int i = 0; i < pDataCols->numOfCols; ++i) { // TODO: dataColAppendVal may fail SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) { + if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter, pDataCols->bitmapMode) < 0) { TASSERT(0); } - tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints); + tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints, pTarget->bitmapMode); } if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++; @@ -1711,7 +1704,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE); } - (*iter)++; + ++(*iter); tSkipListIterNext(pCommitIter->pIter); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index a2dab19141b8da28d2208db4ada03466a2709304..6f96aff84820a9d92111ab545a7743fed570cfd8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -406,7 +406,7 @@ int tsdbUpdateDFileHeader(SDFile *pDFile) { } void *ptr = buf; - taosEncodeFixedU32(&ptr, 0); + // taosEncodeFixedU32(&ptr, 0); // fver moved to SDFInfo and saved to current tsdbEncodeDFInfo(&ptr, &(pDFile->info)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); @@ -437,7 +437,7 @@ int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) { } void *pBuf = buf; - pBuf = taosDecodeFixedU32(pBuf, &_version); + // pBuf = taosDecodeFixedU32(pBuf, &_version); pBuf = tsdbDecodeDFInfo(pBuf, pInfo); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index de6a05df46038360d016347031c82dbc5e01657a..68b86028cf430607c7ad5f86c27609e03017f4c8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1112,7 +1112,7 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle))); + int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)), true); if (ret != TSDB_CODE_SUCCESS) { int32_t c = terrno; assert(c != TSDB_CODE_SUCCESS); @@ -1401,7 +1401,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t // memmove(pData, (char*)src->pData + bytes * start, bytes * num); for(int32_t k = start; k < num + start; ++k) { SCellVal sVal = {0}; - if (tdGetColDataOfRow(&sVal, src, k) < 0) { + if (tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0) { TASSERT(0); } @@ -1415,7 +1415,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t // todo refactor, only copy one-by-one for (int32_t k = start; k < num + start; ++k) { SCellVal sVal = {0}; - if(tdGetColDataOfRow(&sVal, src, k) < 0){ + if(tdGetColDataOfRow(&sVal, src, k, pCols->bitmapMode) < 0){ TASSERT(0); } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 8b718dbb0e201b2e1a203191f71fd6f31ef76910..87459593b5604c66689199fbe9d10f18959d6114 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -276,7 +276,7 @@ int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) { } int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, const int16_t *colIds, - int numOfColsIds) { + int numOfColsIds, bool mergeBitmap) { ASSERT(pBlock->numOfSubBlocks > 0); int8_t update = pReadh->pRepo->config.update; @@ -298,6 +298,16 @@ int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, return -1; } + if (mergeBitmap && !tdDataColsIsBitmapI(pReadh->pDCols[0])) { + for (int i = 0; i < numOfColsIds; ++i) { + SDataCol *pDataCol = pReadh->pDCols[0]->cols + i; + if (pDataCol->bitmap) { + ASSERT(pDataCol->colId != PRIMARYKEY_TIMESTAMP_COL_ID); + tdMergeBitmap(pDataCol->pBitmap, TD_BITMAP_BYTES(pReadh->pDCols[0]->numOfRows), pDataCol->pBitmap); + } + } + } + ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows); ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst); ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast); @@ -499,6 +509,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); tdResetDataCols(pDataCols); + + if(tsdbIsSupBlock(pBlock)) { + tdDataColsSetBitmapI(pDataCols); + } + if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1; SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh); @@ -692,6 +707,10 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * tdResetDataCols(pDataCols); + if(tsdbIsSupBlock(pBlock)) { + tdDataColsSetBitmapI(pDataCols); + } + // If only load timestamp column, no need to load SBlockData part if (numOfColIds > 1 && tsdbLoadBlockOffset(pReadh, pBlock) < 0) return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 902a2d8d35c01eafb30b290ae946499a60c2f645..07b7d6216543b957d517d4833bf51010f487ba66 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -465,16 +465,16 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t * @return int32_t */ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg) { + if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) { + tsdbTrace("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb)); + return TSDB_CODE_SUCCESS; + } + if (!pTsdb->pMeta) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - if (atomic_load_16(&REPO_TSMA_NUM(pTsdb)) <= 0) { - tsdbWarn("vgId:%d not update expire window since no tSma", REPO_ID(pTsdb)); - return TSDB_CODE_SUCCESS; - } - if (tdScanAndConvertSubmitMsg(pMsg) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; }