Skip to content

T
TDengine

taosdata / TDengine
接近 2 年 前同步成功

通知 1191
Star 22018
Fork 4786
T
TDengine
提交 82a227b9 编写于 作者: H Hongze Cheng
浏览文件
操作

more fix

上级 12371d34
隐藏空白更改
内联 并排
Showing with 89 addition and 34 deletion
include/util/tcoding.h
浏览文件 @ 82a227b9
...@@ -23,8 +23,8 @@ extern "C" { ...@@ -23,8 +23,8 @@ extern "C" {
#endif #endif
#define ENCODE_LIMIT (((uint8_t)1) << 7) #define ENCODE_LIMIT (((uint8_t)1) << 7)
#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
/* ------------------------ LEGACY CODES ------------------------ */ /* ------------------------ LEGACY CODES ------------------------ */
#if 1 #if 1
...@@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { ...@@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
} }
static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
*value = ( (((int8_t *)buf)[0] == 0) ? false : true ); *value = ((((int8_t *)buf)[0] == 0) ? false : true);
return POINTER_SHIFT(buf, sizeof(int8_t)); return POINTER_SHIFT(buf, sizeof(int8_t));
} }
......
source/util/src/tcompression.c
浏览文件 @ 82a227b9
...@@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1; ...@@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1;
#define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL)
#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
#ifdef TD_TSZ #ifdef TD_TSZ
bool lossyFloat = false; bool lossyFloat = false;
...@@ -1025,10 +1025,16 @@ static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprA ...@@ -1025,10 +1025,16 @@ static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprA
static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static FORCE_INLINE int64_t tI8ToI64(const void *pData) { return *(int8_t *)pData; } static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; }
static FORCE_INLINE int64_t tI16ToI64(const void *pData) { return *(int16_t *)pData; } static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; }
static FORCE_INLINE int64_t tI32ToI64(const void *pData) { return *(int32_t *)pData; } static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; }
static FORCE_INLINE int64_t tI64ToI64(const void *pData) { return *(int64_t *)pData; } static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; }
static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; }
static struct { static struct {
int8_t type; int8_t type;
int32_t bytes; int32_t bytes;
...@@ -1037,6 +1043,7 @@ static struct { ...@@ -1037,6 +1043,7 @@ static struct {
int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData);
int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *);
int64_t (*getI64)(const void *pData); int64_t (*getI64)(const void *pData);
void (*putI64)(int64_t v, void *pData);
} DATA_TYPE_INFO[] = { } DATA_TYPE_INFO[] = {
{.type = TSDB_DATA_TYPE_NULL, {.type = TSDB_DATA_TYPE_NULL,
.bytes = 0, .bytes = 0,
...@@ -1044,140 +1051,160 @@ static struct { ...@@ -1044,140 +1051,160 @@ static struct {
.startFn = NULL, .startFn = NULL,
.cmprFn = NULL, .cmprFn = NULL,
.endFn = NULL, .endFn = NULL,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_BOOL, {.type = TSDB_DATA_TYPE_BOOL,
.bytes = 1, .bytes = 1,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompBoolStart, .startFn = tCompBoolStart,
.cmprFn = tCompBool, .cmprFn = tCompBool,
.endFn = tCompBoolEnd, .endFn = tCompBoolEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_TINYINT, {.type = TSDB_DATA_TYPE_TINYINT,
.bytes = 1, .bytes = 1,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI8ToI64}, .getI64 = tGetI64OfI8,
.putI64 = tPutI64OfI8},
{.type = TSDB_DATA_TYPE_SMALLINT, {.type = TSDB_DATA_TYPE_SMALLINT,
.bytes = 2, .bytes = 2,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI16ToI64}, .getI64 = tGetI64OfI16,
.putI64 = tPutI64OfI16},
{.type = TSDB_DATA_TYPE_INT, {.type = TSDB_DATA_TYPE_INT,
.bytes = 4, .bytes = 4,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI32ToI64}, .getI64 = tGetI64OfI32,
.putI64 = tPutI64OfI32},
{.type = TSDB_DATA_TYPE_BIGINT, {.type = TSDB_DATA_TYPE_BIGINT,
.bytes = 8, .bytes = 8,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI64ToI64}, .getI64 = tGetI64OfI64,
.putI64 = tPutI64OfI64},
{.type = TSDB_DATA_TYPE_FLOAT, {.type = TSDB_DATA_TYPE_FLOAT,
.bytes = 4, .bytes = 4,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompFloatStart, .startFn = tCompFloatStart,
.cmprFn = tCompFloat, .cmprFn = tCompFloat,
.endFn = tCompFloatEnd, .endFn = tCompFloatEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_DOUBLE, {.type = TSDB_DATA_TYPE_DOUBLE,
.bytes = 8, .bytes = 8,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompDoubleStart, .startFn = tCompDoubleStart,
.cmprFn = tCompDouble, .cmprFn = tCompDouble,
.endFn = tCompDoubleEnd, .endFn = tCompDoubleEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_VARCHAR, {.type = TSDB_DATA_TYPE_VARCHAR,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_TIMESTAMP, {.type = TSDB_DATA_TYPE_TIMESTAMP,
.bytes = 8, .bytes = 8,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompTimestampStart, .startFn = tCompTimestampStart,
.cmprFn = tCompTimestamp, .cmprFn = tCompTimestamp,
.endFn = tCompTimestampEnd, .endFn = tCompTimestampEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_NCHAR, {.type = TSDB_DATA_TYPE_NCHAR,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_UTINYINT, {.type = TSDB_DATA_TYPE_UTINYINT,
.bytes = 1, .bytes = 1,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI8ToI64}, .getI64 = tGetI64OfI8,
.putI64 = tPutI64OfI8},
{.type = TSDB_DATA_TYPE_USMALLINT, {.type = TSDB_DATA_TYPE_USMALLINT,
.bytes = 2, .bytes = 2,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI16ToI64}, .getI64 = tGetI64OfI16,
.putI64 = tPutI64OfI16},
{.type = TSDB_DATA_TYPE_UINT, {.type = TSDB_DATA_TYPE_UINT,
.bytes = 4, .bytes = 4,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI32ToI64}, .getI64 = tGetI64OfI32,
.putI64 = tPutI64OfI32},
{.type = TSDB_DATA_TYPE_UBIGINT, {.type = TSDB_DATA_TYPE_UBIGINT,
.bytes = 8, .bytes = 8,
.isVarLen = 0, .isVarLen = 0,
.startFn = tCompIntStart, .startFn = tCompIntStart,
.cmprFn = tCompInt, .cmprFn = tCompInt,
.endFn = tCompIntEnd, .endFn = tCompIntEnd,
.getI64 = tI64ToI64}, .getI64 = tGetI64OfI64,
.putI64 = tPutI64OfI64},
{.type = TSDB_DATA_TYPE_JSON, {.type = TSDB_DATA_TYPE_JSON,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_VARBINARY, {.type = TSDB_DATA_TYPE_VARBINARY,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_DECIMAL, {.type = TSDB_DATA_TYPE_DECIMAL,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_BLOB, {.type = TSDB_DATA_TYPE_BLOB,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_MEDIUMBLOB, {.type = TSDB_DATA_TYPE_MEDIUMBLOB,
.bytes = 1, .bytes = 1,
.isVarLen = 1, .isVarLen = 1,
.startFn = tCompBinaryStart, .startFn = tCompBinaryStart,
.cmprFn = tCompBinary, .cmprFn = tCompBinary,
.endFn = tCompBinaryEnd, .endFn = tCompBinaryEnd,
.getI64 = NULL}, .getI64 = NULL,
.putI64 = NULL},
}; };
struct SCompressor { struct SCompressor {
...@@ -1443,7 +1470,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { ...@@ -1443,7 +1470,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) {
uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1);
vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev;
memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf);
nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes;
} }
} }
...@@ -1504,8 +1531,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ...@@ -1504,8 +1531,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; pCmprsor->i_end = (pCmprsor->i_end + 1) % 241;
break; break;
} else { } else {
while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
pCmprsor->i_selector++; int32_t lidx = pCmprsor->i_selector + 1;
int32_t ridx = 15;
while (lidx <= ridx) {
pCmprsor->i_selector = (lidx + ridx) >> 1;
if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
lidx = pCmprsor->i_selector + 1;
} else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
ridx = pCmprsor->i_selector - 1;
} else {
break;
}
}
if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
} }
nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
...@@ -1551,8 +1592,22 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ ...@@ -1551,8 +1592,22 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_
int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241;
if (nEle == 0) break; if (nEle == 0) break;
while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
pCmprsor->i_selector++; int32_t lidx = pCmprsor->i_selector + 1;
int32_t ridx = 15;
while (lidx <= ridx) {
pCmprsor->i_selector = (lidx + ridx) >> 1;
if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
lidx = pCmprsor->i_selector + 1;
} else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
ridx = pCmprsor->i_selector - 1;
} else {
break;
}
}
if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
} }
nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或