diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index da724206d9d01c7cf61ac57ca7c27a298ddff85b..991cb55e5070a55470278554a73e4f49c3e32184 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -135,8 +135,8 @@ typedef struct { #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 #define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) -#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) +#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS)) +#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8)) int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); @@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche } #endif // ----------------- Data column structure +// SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { int8_t type; // column type uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM diff --git a/include/common/trow.h b/include/common/trow.h index 01f40763829d7a3edd01a7bd01e59883ff566ac4..7cde8c50c5825766d78c2498a55db79e162dcbc6 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -147,20 +147,20 @@ typedef struct { typedef struct { // basic info - int8_t rowType; - int16_t sver; - STSRow *pBuf; + int8_t rowType; + schema_ver_t sver; + STSRow *pBuf; // extended info - int32_t flen; - int16_t nBoundCols; - int16_t nCols; - int16_t nBitmaps; - int16_t nBoundBitmaps; - int32_t offset; - void *pBitmap; - void *pOffset; - int32_t extendedRowSize; + int32_t flen; + col_id_t nBoundCols; + col_id_t nCols; + col_id_t nBitmaps; + col_id_t nBoundBitmaps; + int32_t offset; + void *pBitmap; + void *pOffset; + int32_t extendedRowSize; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); } else { pBuilder->nBoundBitmaps = 0; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1d7a4aeb8fc44e3336e33e6cdc31f4d477157817..c1225012ac63baf3c489e0525504b6fca9e5b352 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - spaceNeeded += TYPE_BYTES[pCol->type]; + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove + // the additional space #endif if (pCol->spaceSize < spaceNeeded) { @@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { } } #ifdef TD_SUPPORT_BITMAP + if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints); pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes); @@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) bool isNEleNull(SDataCol *pCol, int nEle) { if (isAllRowsNull(pCol)) return true; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; @@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { dataColSetNullAt(pCol, i); } } else { @@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { // char *tptr = (char *)(pCol->pData); VarDataOffsetT offset = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { pCol->dataOff[i] = offset; offset += varDataTLen(tptr); tptr = POINTER_SHIFT(tptr, varDataTLen(tptr)); @@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { tdFreeDataCols(pCols); return NULL; } +#if 0 // no need as calloc used int i; for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; @@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } +#endif } return pCols; @@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for (i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; ++i) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; + pCols->cols[i].pBitmap = NULL; pCols->cols[i].spaceSize = 0; } } +#if 0 + tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...) +#endif - tdResetDataCols(pCols); + pCols->numOfRows = 0; pCols->numOfCols = schemaNCols(pSchema); - for (i = 0; i < schemaNCols(pSchema); i++) { + for (i = 0; i < schemaNCols(pSchema); ++i) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } @@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { if (pCols->cols) { int maxCols = pCols->maxCols; - for (i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; ++i) { SDataCol *pCol = &pCols->cols[i]; taosMemoryFreeClear(pCol->pData); } @@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; - for (int i = 0; i < pCols->maxCols; i++) { + for (int i = 0; i < pCols->maxCols; ++i) { dataColReset(pCols->cols + i); } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9ee2ec1300487b5a46e3a63a2f8b8881b639b2ab..0d5a874c1c8a0eaf1915a7afb25a96b154b91855 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); } } diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 17c220a35a30b97b4989cf7bd6dc1c84643e937a..90a877bb83ef77050cb12637c0fdcb083ee075f7 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -112,10 +112,10 @@ typedef struct { #else typedef struct { int16_t colId; - uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint8_t reserve : 7; - uint8_t type; - int32_t len; + uint16_t type : 6; + uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) + uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t len : 31; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d5e9b55a7167818d75a3b7a8935c89ecc0c9f968..497bae6ee150ac3292df7775fc036544016880f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish @@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; + int32_t tBitmapsLen = 0; if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tBitmaps = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tlen += tBitmaps; @@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif // move bitmap parts ahead // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); } #endif void *tptr; // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); @@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF // Compress or just copy if (pCfg->compression) { +#if 0 flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); +#endif + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + tBitmapsLen = + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, rowsToWrite, tptr, tBitmaps + COMP_OVERFLOW_BYTES, + pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); + TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); + flen += tBitmapsLen; + } } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(tptr, pDataCol->pBitmap, tBitmaps); + tBitmapsLen = tBitmaps; + flen += tBitmapsLen; + } } // Add checksum ASSERT(flen > 0); + ASSERT(tBitmapsLen <= 1024); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; + pBlockCol->len = flen; // data + bitmaps + pBlockCol->blen = tBitmapsLen; ++tcol; } else { keyLen = flen; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 304b3286fe31e2c796c7f68cd0f98df6f4741a87..8cb90d76a63b36ffd8d7bb4c03bc38b7a2e43e10 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, int numOfColIds); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); @@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; continue; } @@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat TD_SET_COL_ROWS_NORM(pDataCol); } - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } if (tcolId == pDataCol->colId) { if (pBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), - (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen, + pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints, + TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return 0; } -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize) { +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) { if (!taosCheckChecksumWhole((uint8_t *)content, len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; @@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 if (comp) { // Need to decompress int tlen = - (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps, + (*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); + tsdbError( + "Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + (int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } pDataCol->len = tlen; + + if (numOfBitmaps > 0) { + tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps, + pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError( + "Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d " + "bufferSize:%d", + bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + // pDataCol->blen = tlen; + } } else { // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); + pDataCol->len = len - bitmapLen - sizeof(TSCKSUM); memcpy(pDataCol->pData, content, pDataCol->len); + if (numOfBitmaps > 0) { + // pDataCol->blen = bitmapLen; + memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen); + } } +#if 0 if (lenOfBitmaps > 0) { pDataCol->len -= lenOfBitmaps; @@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } else if (IS_VAR_DATA_TYPE(pDataCol->type)) { dataColSetOffset(pDataCol, numOfRows); } - +#endif + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } return 0; } @@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb * pRepo = TSDB_READ_REPO(pReadh); + STsdb *pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); - int32_t tBitmaps = 0; + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if (!TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; @@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, - tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm, + pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), pBlockCol->colId, offset); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b2332aded7fd79beb8feb6539ddf8a1200de4a5b..041aec4054761b52e926f874401f9acca5bfb27f 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -28,11 +28,11 @@ * * BOOLEAN Compression Algorithm: * We provide two methods for compress boolean types. Because boolean types in C - * code are char bytes with 0 and 1 values only, only one bit can used to discrimenate + * code are char bytes with 0 and 1 values only, only one bit can used to discriminate * the values. * 1. The first method is using only 1 bit to represent the boolean value with 1 for * true and 0 for false. Then the compression rate is 1/8. - * 2. The second method is using run length encoding (RLE) methods. This methos works + * 2. The second method is using run length encoding (RLE) methods. This method works * better when there are a lot of consecutive true values or false values. * * STRING Compression Algorithm: