diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index b01b7ac24064495775910ab97fe91a10f4d1b749..de7c2a49f605454d3b8368deb3715957605b6056 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -33,9 +33,9 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct SKVIdx SKVIdx; -// STSchema - // STSRow2 +#define TSROW_SVER(r) (((r)->flags & TSROW_KV_ROW) ? -1 : (r)->sver) + int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, @@ -70,9 +70,13 @@ struct STSchema { STColumn columns[]; }; +#define TSROW_HAS_NONE ((uint8_t)0x1) +#define TSROW_HAS_NULL ((uint8_t)0x2U) +#define TSROW_HAS_VAL ((uint8_t)0x4U) +#define TSROW_KV_ROW ((uint8_t)0x10U) struct STSRow2 { - TSKEY ts; - uint32_t flags; + TSKEY ts; + uint8_t flags; union { int32_t sver; int32_t ncols; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8eb9ae62d8cea476ac508d640ad7efcf26f5e21d..d860ce5b996efe75343c0e4b6622d3133ac5dd97 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -275,9 +275,8 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_SMA_ON ((int8_t)0x1) #define COL_IDX_ON ((int8_t)0x2) -#define COL_SET_VAL ((int8_t)0x10) -#define COL_SET_NONE ((int8_t)0x20) -#define COL_SET_NULL ((int8_t)0x40) +#define COL_SET_NULL ((int8_t)0x10) +#define COL_SET_VAL ((int8_t)0x20) typedef struct SSchema { int8_t type; int8_t flags; @@ -286,7 +285,8 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0) +#define COL_IS_SET(FLG) ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0) +#define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d492d98b63c4b8108a1bf8d6825efdd8b3b3221b..0fed7c86feaa457a457dd23c67861595b0581c31 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,38 +19,66 @@ #include "tdatablock.h" #include "tlog.h" -#define TD_HAS_NONE 0x1U -#define TD_HAS_NULL 0x2U -#define TD_HAS_VAL 0x4U -#define TD_KV_ROW 0x10U - struct SKVIdx { int32_t cid; int32_t offset; }; +#define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) + // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; - if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; - } else { - if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + if (tEncodeU8(pEncoder, pRow->flags) < 0) return -1; + + ASSERT(pRow->flags & 0xf != 0); + + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + ASSERT(TSROW_IS_KV_ROW(pRow) && pRow->nData == 0 && pRow->pData == NULL); + return 0; + case TSROW_HAS_VAL: + ASSERT(!TSROW_IS_KV_ROW(pRow)); + default: + ASSERT(pRow->nData && pRow->pData); + if (TSROW_IS_KV_ROW(pRow)) { + if (tEncodeI32v(pEncoder, pRow->ncols) < 0) return -1; + } else { + if (tEncodeI32v(pEncoder, pRow->sver) < 0) return -1; + } + if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData)) return -1; + break; } - if (tEncodeBinary(pEncoder, pRow->pData, pRow->nData) < 0) return -1; + return 0; } int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { if (tDecodeI64(pDecoder, &pRow->ts) < 0) return -1; - if (tDecodeU32v(pDecoder, &pRow->flags) < 0) return -1; - if (pRow->flags & TD_KV_ROW) { - if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; - } else { - if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + if (tDecodeU8(pDecoder, &pRow->flags) < 0) return -1; + + ASSERT(pRow->flags & 0xf != 0); + + switch (pRow->flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + ASSERT(TSROW_IS_KV_ROW(pRow)); + pRow->nData = 0; + pRow->pData = NULL; + return 0; + case TSROW_HAS_VAL: + ASSERT(!TSROW_IS_KV_ROW(pRow)); + default: + if (TSROW_IS_KV_ROW(pRow)) { + if (tDecodeI32v(pDecoder, &pRow->ncols) < 0) return -1; + } else { + if (tDecodeI32v(pDecoder, &pRow->sver) < 0) return -1; + } + if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData)) return -1; + break; } - if (tDecodeBinary(pDecoder, &pRow->pData, &pRow->nData) < 0) return -1; + return 0; } @@ -66,17 +94,17 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui *nData = 0; switch (tflags) { - case TD_HAS_NONE: + case TSROW_HAS_NONE: *flags = -1; break; - case TD_HAS_NULL: + case TSROW_HAS_NULL: *flags = 1; break; - case TD_HAS_VAL: + case TSROW_HAS_VAL: *flags = 0; // find the row break; - case TD_HAS_NULL | TD_HAS_NONE: + case TSROW_HAS_NULL | TSROW_HAS_NONE: // read bit map (todo) if (0) { *flags = 1; @@ -84,11 +112,11 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui *flags = -1; } break; - case TD_HAS_VAL | TD_HAS_NONE: - case TD_HAS_VAL | TD_HAS_NULL: + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: // read bitmap (todo) if (0) { - if (tflags & TD_HAS_NONE) { + if (tflags & TSROW_HAS_NONE) { *flags = -1; } else { *flags = 1; @@ -97,7 +125,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const ui // get value (todo) } break; - case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: break; default: return -1; @@ -199,18 +227,15 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { } int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pData, uint32_t nData) { - int32_t iCol; - uint8_t *p; - - // search column (todo: make here faster) + // search column (TODO: bsearch with interp) if (pBuilder->pTColumn->colId < cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; + int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; if (pBuilder->pTColumn->colId == cid) break; } } else if (pBuilder->pTColumn->colId > cid) { - iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; + int32_t iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) - 1; for (; iCol >= 0; iCol--) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; if (pBuilder->pTColumn->colId == cid) break; @@ -218,19 +243,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD } // check - if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) { + if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn->flags)) { return -1; } // set value + uint8_t *p; if (cid == 0) { ASSERT(pData && nData == sizeof(TSKEY)); pBuilder->row.ts = *(TSKEY *)pData; pBuilder->pTColumn->flags |= COL_SET_VAL; } else { - if (pData) { - pBuilder->row.flags |= TD_HAS_VAL; + if (pData) { // set val + pBuilder->row.flags |= TSROW_HAS_VAL; pBuilder->pTColumn->flags |= COL_SET_VAL; // set tuple data @@ -250,20 +276,20 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD ((SKVIdx *)p)->cid = cid; ((SKVIdx *)p)->offset = pBuilder->kvVLen; - p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols + pBuilder->kvVLen; + p = pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1) + pBuilder->kvVLen; if (IS_VAR_DATA_TYPE(pBuilder->pTColumn->type)) { pBuilder->kvVLen += tPutBinary(p, pData, nData); } else { memcpy(p, pData, nData); pBuilder->kvVLen += nData; } - } else { - pBuilder->row.flags |= TD_HAS_NULL; + } else { // set NULL + pBuilder->row.flags |= TSROW_HAS_NULL; pBuilder->pTColumn->flags |= COL_SET_NULL; p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; ((SKVIdx *)p)->cid = cid; - ((SKVIdx *)p)->offset = -1; + ((SKVIdx *)p)->offset = -1; // for TSROW_KV_ROW, offset -1 means NULL } pBuilder->nCols++; @@ -272,68 +298,76 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD return 0; } +static FORCE_INLINE int tSKVIdxCmprFn(const void *p1, const void *p2) { + SKVIdx *pKVIdx1 = (SKVIdx *)p1; + SKVIdx *pKVIdx2 = (SKVIdx *)p2; + if (pKVIdx1->cid > pKVIdx2->cid) { + return 1; + } else if (pKVIdx1->cid < pKVIdx2->cid) { + return -1; + } + return 0; +} int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { int32_t tpDataLen, kvDataLen; uint32_t flags; // error not set ts - if (!COL_IS_SET(pBuilder->pTSchema->columns)) { + if (!COL_IS_SET(pBuilder->pTSchema->columns->flags)) { return -1; } + ASSERT(pBuilder->nCols < pBuilder->pTSchema->numOfCols); if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { - pBuilder->row.flags |= TD_HAS_NONE; + pBuilder->row.flags |= TSROW_HAS_NONE; } - flags = pBuilder->row.flags & 0xf; - switch (flags) { - case TD_HAS_NONE: - case TD_HAS_NULL: - pBuilder->row.sver = pBuilder->pTSchema->version; + ASSERT(pBuilder->row.flags & 0xf != 0); + *(ppRow) = &pBuilder->row; + switch (pBuilder->row.flags & 0xf) { + case TSROW_HAS_NONE: + case TSROW_HAS_NULL: + pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = 0; pBuilder->row.pData = NULL; + return 0; + case TSROW_HAS_NULL | TSROW_HAS_NONE: + tpDataLen = (pBuilder->pTSchema->numOfCols - 1) / 8; break; - case TD_HAS_VAL: - pBuilder->row.sver = pBuilder->pTSchema->version; - pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTPBuf; + case TSROW_HAS_VAL: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen; break; - case TD_HAS_NULL | TD_HAS_NONE: - pBuilder->row.sver = pBuilder->pTSchema->version; - // set bitmap (todo) - pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1; - pBuilder->row.pData = pBuilder->pBitBuf; + case TSROW_HAS_VAL | TSROW_HAS_NONE: + case TSROW_HAS_VAL | TSROW_HAS_NULL: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 8; break; - case TD_HAS_VAL | TD_HAS_NONE: - case TD_HAS_VAL | TD_HAS_NULL: - case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: - if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) { - tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; - } else { - tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; - } - kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; - - if (kvDataLen < tpDataLen) { - pBuilder->row.flags |= TD_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; - pBuilder->row.nData = kvDataLen; - pBuilder->row.pData = pBuilder->pKVBuf; - // memmove(); todo - // qsort - } else { - pBuilder->row.sver = pBuilder->pTSchema->numOfCols; - // set bitmap etc (todo) - pBuilder->row.nData = tpDataLen; - pBuilder->row.pData = pBuilder->pTPBuf; - } + case TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE: + tpDataLen = pBuilder->pTSchema->flen + pBuilder->tpVLen + (pBuilder->pTSchema->numOfCols - 1) / 4; break; default: - ASSERT(0); - return -1; + // decide chose tuple or kv + kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; + break; + } + + if (kvDataLen < tpDataLen) { + pBuilder->row.flags |= TSROW_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + + pBuilder->row.nData = kvDataLen; + pBuilder->row.pData = pBuilder->pKVBuf; + qsort(pBuilder->pKVBuf, pBuilder->nCols, sizeof(SKVIdx), tSKVIdxCmprFn); + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { + memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, + pBuilder->pKVBuf + sizeof(SKVIdx) * (pBuilder->pTSchema->numOfCols - 1), pBuilder->kvVLen); + } + } else { + pBuilder->row.sver = pBuilder->pTSchema->version; + + pBuilder->row.nData = tpDataLen; + pBuilder->row.pData } - *ppRow = &pBuilder->row; return 0; }