From 582176a4c1cedef4e7d7c00ed8dc5c4c8e2fba98 Mon Sep 17 00:00:00 2001 From: hzcheng Date: Thu, 7 May 2020 09:31:46 +0800 Subject: [PATCH] add encoding functions --- src/common/inc/tdataformat.h | 8 +-- src/common/src/tdataformat.c | 22 +++--- src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMeta.c | 2 +- src/util/inc/tcoding.h | 131 ++++++++++++++++++++++++----------- src/util/inc/tutil.h | 2 +- 6 files changed, 108 insertions(+), 58 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 10ca75f785..ba37db519e 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -114,10 +114,10 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o switch (type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - return POINTER_DRIFT(row, *(VarDataOffsetT *)POINTER_DRIFT(row, offset)); + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); break; default: - return POINTER_DRIFT(row, offset); + return POINTER_SHIFT(row, offset); break; } } @@ -149,11 +149,11 @@ static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { switch (pCol->type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - return POINTER_DRIFT(pCol->pData, pCol->dataOff[row]); + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); break; default: - return POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); break; } } diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 28f42f85a5..9d81cd07af 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -167,17 +167,17 @@ void tdFreeDataRow(SDataRow row) { int tdAppendColVal(SDataRow row, void *value, int8_t type, int32_t bytes, int32_t offset) { ASSERT(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; - char * ptr = POINTER_DRIFT(row, dataRowLen(row)); + char * ptr = POINTER_SHIFT(row, dataRowLen(row)); switch (type) { case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)POINTER_DRIFT(row, toffset) = dataRowLen(row); + *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); memcpy(ptr, value, varDataTLen(value)); dataRowLen(row) += varDataTLen(value); break; default: - memcpy(POINTER_DRIFT(row, toffset), value, TYPE_BYTES[type]); + memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); break; } @@ -202,13 +202,13 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { pDataCol->spaceSize = (sizeof(VarDataLenT) + pDataCol->bytes) * maxPoints; pDataCol->dataOff = (VarDataOffsetT *)(*pBuf); - pDataCol->pData = POINTER_DRIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints); - *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints); + pDataCol->pData = POINTER_SHIFT(*pBuf, TYPE_BYTES[pDataCol->type] * maxPoints); + *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize + TYPE_BYTES[pDataCol->type] * maxPoints); } else { pDataCol->spaceSize = pDataCol->bytes * maxPoints; pDataCol->dataOff = NULL; pDataCol->pData = *pBuf; - *pBuf = POINTER_DRIFT(*pBuf, pDataCol->spaceSize); + *pBuf = POINTER_SHIFT(*pBuf, pDataCol->spaceSize); } } @@ -222,13 +222,13 @@ void dataColAppendVal(SDataCol *pCol, void *value, int numOfPoints, int maxPoint // set offset pCol->dataOff[numOfPoints] = pCol->len; // Copy data - memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); // Update the length pCol->len += varDataTLen(value); break; default: ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints); - memcpy(POINTER_DRIFT(pCol->pData, pCol->len), value, pCol->bytes); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; break; } @@ -244,12 +244,12 @@ void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfPoints) { VarDataOffsetT toffset = pCol->dataOff[pointsToPop]; pCol->len = pCol->len - toffset; ASSERT(pCol->len > 0); - memmove(pCol->pData, POINTER_DRIFT(pCol->pData, toffset), pCol->len); + memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len); dataColSetOffset(pCol, pointsLeft); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfPoints); pCol->len = TYPE_BYTES[pCol->type] * pointsLeft; - memmove(pCol->pData, POINTER_DRIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len); + memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len); } } @@ -301,7 +301,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { for (int i = 0; i < nEle; i++) { pCol->dataOff[i] = offset; offset += varDataTLen(tptr); - tptr = POINTER_DRIFT(tptr, varDataTLen(tptr)); + tptr = POINTER_SHIFT(tptr, varDataTLen(tptr)); } } diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index e056a10bba..a681c865c9 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -21,6 +21,7 @@ #include "tskiplist.h" #include "tutil.h" #include "tlog.h" +#include "tcoding.h" #ifdef __cplusplus extern "C" { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index ec4fb2a33f..63ccb538ed 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -535,5 +535,5 @@ static int tsdbEstimateTableEncodeSize(STable *pTable) { char *getTupleKey(const void * data) { SDataRow row = (SDataRow)data; - return POINTER_DRIFT(row, TD_DATA_ROW_HEAD_SIZE); + return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); } \ No newline at end of file diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index f3e7a52942..e74222cb9b 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -24,96 +24,145 @@ extern "C" { #include "tutil.h" -const int TNUMBER = 1; -#define IS_LITTLE_ENDIAN() (*(char *)(&TNUMBER) != 0) +const int32_t TNUMBER = 1; +const uint8_t ENCODE_LIMIT = (1 << 7); +#define IS_LITTLE_ENDIAN() (*(uint8_t *)(&TNUMBER) != 0) static FORCE_INLINE void *taosEncodeFixed16(void *buf, uint16_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { - ((char *)buf)[0] = value & 0xff; - ((char *)buf)[1] = (value >> 8) & 0xff; + ((uint8_t *)buf)[0] = value & 0xff; + ((uint8_t *)buf)[1] = (value >> 8) & 0xff; } - return POINTER_DRIFT(buf, sizeof(value)); + return POINTER_SHIFT(buf, sizeof(value)); } static FORCE_INLINE void *taosEncodeFixed32(void *buf, uint32_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { - ((char *)buf)[0] = value & 0xff; - ((char *)buf)[1] = (value >> 8) & 0xff; - ((char *)buf)[2] = (value >> 16) & 0xff; - ((char *)buf)[3] = (value >> 24) & 0xff; + ((uint8_t *)buf)[0] = value & 0xff; + ((uint8_t *)buf)[1] = (value >> 8) & 0xff; + ((uint8_t *)buf)[2] = (value >> 16) & 0xff; + ((uint8_t *)buf)[3] = (value >> 24) & 0xff; } - return POINTER_DRIFT(buf, sizeof(value)); + return POINTER_SHIFT(buf, sizeof(value)); } static FORCE_INLINE void *taosEncodeFixed64(void *buf, uint64_t value) { if (IS_LITTLE_ENDIAN()) { memcpy(buf, &value, sizeof(value)); } else { - ((char *)buf)[0] = value & 0xff; - ((char *)buf)[1] = (value >> 8) & 0xff; - ((char *)buf)[2] = (value >> 16) & 0xff; - ((char *)buf)[3] = (value >> 24) & 0xff; - ((char *)buf)[4] = (value >> 32) & 0xff; - ((char *)buf)[5] = (value >> 40) & 0xff; - ((char *)buf)[6] = (value >> 48) & 0xff; - ((char *)buf)[7] = (value >> 56) & 0xff; + ((uint8_t *)buf)[0] = value & 0xff; + ((uint8_t *)buf)[1] = (value >> 8) & 0xff; + ((uint8_t *)buf)[2] = (value >> 16) & 0xff; + ((uint8_t *)buf)[3] = (value >> 24) & 0xff; + ((uint8_t *)buf)[4] = (value >> 32) & 0xff; + ((uint8_t *)buf)[5] = (value >> 40) & 0xff; + ((uint8_t *)buf)[6] = (value >> 48) & 0xff; + ((uint8_t *)buf)[7] = (value >> 56) & 0xff; } - return POINTER_DRIFT(buf, sizeof(value)); + return POINTER_SHIFT(buf, sizeof(value)); } static FORCE_INLINE void *taosDecodeFixed16(void *buf, uint16_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { - ((char *)value)[1] = ((char *)buf)[0]; - ((char *)value)[0] = ((char *)buf)[1]; + ((uint8_t *)value)[1] = ((uint8_t *)buf)[0]; + ((uint8_t *)value)[0] = ((uint8_t *)buf)[1]; } - return POINTER_DRIFT(buf, sizeof(*value)); + return POINTER_SHIFT(buf, sizeof(*value)); } static FORCE_INLINE void *taosDecodeFixed32(void *buf, uint32_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { - ((char *)value)[3] = ((char *)buf)[0]; - ((char *)value)[2] = ((char *)buf)[1]; - ((char *)value)[1] = ((char *)buf)[2]; - ((char *)value)[0] = ((char *)buf)[3]; + ((uint8_t *)value)[3] = ((uint8_t *)buf)[0]; + ((uint8_t *)value)[2] = ((uint8_t *)buf)[1]; + ((uint8_t *)value)[1] = ((uint8_t *)buf)[2]; + ((uint8_t *)value)[0] = ((uint8_t *)buf)[3]; } - return POINTER_DRIFT(buf, sizeof(*value)); + return POINTER_SHIFT(buf, sizeof(*value)); } static FORCE_INLINE void *taosDecodeFixed64(void *buf, uint64_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { - ((char *)value)[7] = ((char *)buf)[0]; - ((char *)value)[6] = ((char *)buf)[1]; - ((char *)value)[5] = ((char *)buf)[2]; - ((char *)value)[4] = ((char *)buf)[3]; - ((char *)value)[3] = ((char *)buf)[4]; - ((char *)value)[2] = ((char *)buf)[5]; - ((char *)value)[1] = ((char *)buf)[6]; - ((char *)value)[0] = ((char *)buf)[7]; + ((uint8_t *)value)[7] = ((uint8_t *)buf)[0]; + ((uint8_t *)value)[6] = ((uint8_t *)buf)[1]; + ((uint8_t *)value)[5] = ((uint8_t *)buf)[2]; + ((uint8_t *)value)[4] = ((uint8_t *)buf)[3]; + ((uint8_t *)value)[3] = ((uint8_t *)buf)[4]; + ((uint8_t *)value)[2] = ((uint8_t *)buf)[5]; + ((uint8_t *)value)[1] = ((uint8_t *)buf)[6]; + ((uint8_t *)value)[0] = ((uint8_t *)buf)[7]; } - return POINTER_DRIFT(buf, sizeof(*value)); + return POINTER_SHIFT(buf, sizeof(*value)); +} + +static FORCE_INLINE void *taosEncodeVariant16(void *buf, uint16_t value) { + int i = 0; + while (value >= ENCODE_LIMIT) { + ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + value >>= 7; + i++; + ASSERT(i < 3); + } + + ((uint8_t *)buf)[i] = value; + + return POINTER_SHIFT(buf, i+1); +} + +static FORCE_INLINE void *taosEncodeVariant32(void *buf, uint32_t value) { + int i = 0; + while (value >= ENCODE_LIMIT) { + ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + value >>= 7; + i++; + ASSERT(i < 5); + } + + ((uint8_t *)buf)[i] = value; + + return POINTER_SHIFT(buf, i + 1); +} + +static FORCE_INLINE void *taosEncodeVariant64(void *buf, uint64_t value) { + int i = 0; + while (value >= ENCODE_LIMIT) { + ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + value >>= 7; + i++; + ASSERT(i < 10); + } + + ((uint8_t *)buf)[i] = value; + + return POINTER_SHIFT(buf, i + 1); +} + +static FORCE_INLINE void *taosDecodeVariant16(void *buf, uint16_t *value) { + int i = 0; + *value = 0; + while (i < 3 && ) { + (*value) |= (((uint8_t *)buf)[i] << (7 * i)); + i++; + } + + return NULL; // error happened } -// TODO -static FORCE_INLINE void *taosEncodeVariant16(void *buf, uint16_t value) {} -static FORCE_INLINE void *taosEncodeVariant32(void *buf, uint32_t value) {} -static FORCE_INLINE void *taosEncodeVariant64(void *buf, uint64_t value) {} -static FORCE_INLINE void *taosDecodeVariant16(void *buf, uint16_t *value) {} static FORCE_INLINE void *taosDecodeVariant32(void *buf, uint32_t *value) {} static FORCE_INLINE void *taosDecodeVariant64(void *buf, uint64_t *value) {} diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index b03e0a1c5b..9528904882 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -45,7 +45,7 @@ extern "C" { #define tclose(x) taosCloseSocket(x) // Pointer p drift right by b bytes -#define POINTER_DRIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) #ifndef NDEBUG #define ASSERT(x) assert(x) -- GitLab