Skip to content

T
TDengine

taosdata / TDengine
大约 2 年 前同步成功

通知 1192
Star 22018
Fork 4786
T
TDengine
提交 05a39977 编写于 作者: D dapan1121
浏览文件
操作

Merge remote-tracking branch 'origin/3.0' into feat/caseWhen

上级 50b396c8 0575cc26
隐藏空白更改
内联 并排
Showing with 3161 addition and 1562 deletion
include/common/tdataformat.h
浏览文件 @ 05a39977
...@@ -162,17 +162,7 @@ struct STSRowBuilder { ...@@ -162,17 +162,7 @@ struct STSRowBuilder {
struct SValue { struct SValue {
union { union {
int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT int64_t val;
uint8_t u8; // TSDB_DATA_TYPE_UTINYINT
int16_t i16; // TSDB_DATA_TYPE_SMALLINT
uint16_t u16; // TSDB_DATA_TYPE_USMALLINT
int32_t i32; // TSDB_DATA_TYPE_INT
uint32_t u32; // TSDB_DATA_TYPE_UINT
int64_t i64; // TSDB_DATA_TYPE_BIGINT
uint64_t u64; // TSDB_DATA_TYPE_UBIGINT
TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP
float f; // TSDB_DATA_TYPE_FLOAT
double d; // TSDB_DATA_TYPE_DOUBLE
struct { struct {
uint32_t nData; uint32_t nData;
uint8_t *pData; uint8_t *pData;
......
include/common/ttypes.h
浏览文件 @ 05a39977
...@@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor { ...@@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor {
char *name; char *name;
int64_t minValue; int64_t minValue;
int64_t maxValue; int64_t maxValue;
int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t nBuf);
int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t nBuf);
void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min,
int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor; } tDataTypeDescriptor;
...@@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); ...@@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
void *getDataMin(int32_t type); void *getDataMin(int32_t type);
void *getDataMax(int32_t type); void *getDataMax(int32_t type);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
......
include/util/tcoding.h
浏览文件 @ 05a39977
...@@ -23,8 +23,8 @@ extern "C" { ...@@ -23,8 +23,8 @@ extern "C" {
#endif #endif
#define ENCODE_LIMIT (((uint8_t)1) << 7) #define ENCODE_LIMIT (((uint8_t)1) << 7)
#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
/* ------------------------ LEGACY CODES ------------------------ */ /* ------------------------ LEGACY CODES ------------------------ */
#if 1 #if 1
...@@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { ...@@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
} }
static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
*value = ( (((int8_t *)buf)[0] == 0) ? false : true ); *value = ((((int8_t *)buf)[0] == 0) ? false : true);
return POINTER_SHIFT(buf, sizeof(int8_t)); return POINTER_SHIFT(buf, sizeof(int8_t));
} }
......
include/util/tcompression.h
浏览文件 @ 05a39977
...@@ -51,287 +51,12 @@ extern "C" { ...@@ -51,287 +51,12 @@ extern "C" {
#define HEAD_MODE(x) x % 2 #define HEAD_MODE(x) x % 2
#define HEAD_ALGO(x) x / 2 #define HEAD_ALGO(x) x / 2
extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type);
extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output,
const char type);
extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize);
extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output,
int32_t outputSize);
extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output);
extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output);
// lossy
extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output);
extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
char *const output);
extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output);
extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
char *const output);
#ifdef TD_TSZ #ifdef TD_TSZ
extern bool lossyFloat; extern bool lossyFloat;
extern bool lossyDouble; extern bool lossyDouble;
int32_t tsCompressInit(); int32_t tsCompressInit();
void tsCompressExit(); void tsCompressExit();
#endif
static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize,
const int32_t nelements, char *const output, int32_t outputSize,
char algorithm, char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize,
const int32_t nelements, char *const output, int32_t outputSize,
char algorithm, char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsCompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressBoolImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressBoolImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressBoolImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
}
static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
return tsCompressStringImp(input, inputSize, output, outputSize);
}
static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
return tsDecompressStringImp(input, compressedSize, output, outputSize);
}
static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
#ifdef TD_TSZ
// lossy mode
if (lossyFloat) {
return tsCompressFloatLossyImp(input, nelements, output);
// lossless mode
} else {
#endif
if (algorithm == ONE_STAGE_COMP) {
return tsCompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressFloatImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(input, compressedSize, nelements, output);
} else {
#endif
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressFloatImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressFloatImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const buffer,
int32_t bufferSize) {
#ifdef TD_TSZ
if (lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(input, nelements, output);
} else {
#endif
// lossless mode
if (algorithm == ONE_STAGE_COMP) {
return tsCompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
int32_t len = tsCompressDoubleImp(input, nelements, buffer);
return tsCompressStringImp(buffer, len, output, outputSize);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output);
} else {
#endif
// decompress lossless
if (algorithm == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(input, nelements, output);
} else if (algorithm == TWO_STAGE_COMP) {
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
return tsDecompressDoubleImp(buffer, nelements, output);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
#ifdef TD_TSZ
//
// lossy float double
//
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) { char *const buffer, int32_t bufferSize) {
...@@ -358,33 +83,56 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int ...@@ -358,33 +83,56 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int
#endif #endif
static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, /*************************************************************************
char *const output, int32_t outputSize, char algorithm, * REGULAR COMPRESSION
char *const buffer, int32_t bufferSize) { *************************************************************************/
if (algorithm == ONE_STAGE_COMP) { int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return tsCompressTimestampImp(input, nelements, output); int32_t nBuf);
} else if (algorithm == TWO_STAGE_COMP) { int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
int32_t len = tsCompressTimestampImp(input, nelements, buffer); void *pBuf, int32_t nBuf);
return tsCompressStringImp(buffer, len, output, outputSize); int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} else { int32_t nBuf);
assert(0); int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return -1; int32_t nBuf);
} int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} int32_t nBuf);
int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, int32_t nBuf);
const int32_t nelements, char *const output, int32_t outputSize, int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
char algorithm, char *const buffer, int32_t bufferSize) { int32_t nBuf);
if (algorithm == ONE_STAGE_COMP) { int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return tsDecompressTimestampImp(input, nelements, output); int32_t nBuf);
} else if (algorithm == TWO_STAGE_COMP) { int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; int32_t nBuf);
return tsDecompressTimestampImp(buffer, nelements, output); int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} else { int32_t nBuf);
assert(0); int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
return -1; int32_t nBuf);
} int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} int32_t nBuf);
int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf);
/*************************************************************************
* STREAM COMPRESSION
*************************************************************************/
typedef struct SCompressor SCompressor;
int32_t tCompressorCreate(SCompressor **ppCmprsor);
int32_t tCompressorDestroy(SCompressor *pCmprsor);
int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin);
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData);
#ifdef __cplusplus #ifdef __cplusplus
} }
......
source/common/src/tdataformat.c
浏览文件 @ 05a39977
...@@ -33,105 +33,21 @@ typedef struct { ...@@ -33,105 +33,21 @@ typedef struct {
// SValue // SValue
int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); return tPutBinary(p, pValue->pData, pValue->nData);
} else { } else {
switch (type) { if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes);
case TSDB_DATA_TYPE_BOOL: return tDataTypes[type].bytes;
n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0);
break;
case TSDB_DATA_TYPE_TINYINT:
n += tPutI8(p ? p + n : p, pValue->i8);
break;
case TSDB_DATA_TYPE_SMALLINT:
n += tPutI16(p ? p + n : p, pValue->i16);
break;
case TSDB_DATA_TYPE_INT:
n += tPutI32(p ? p + n : p, pValue->i32);
break;
case TSDB_DATA_TYPE_BIGINT:
n += tPutI64(p ? p + n : p, pValue->i64);
break;
case TSDB_DATA_TYPE_FLOAT:
n += tPutFloat(p ? p + n : p, pValue->f);
break;
case TSDB_DATA_TYPE_DOUBLE:
n += tPutDouble(p ? p + n : p, pValue->d);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
n += tPutI64(p ? p + n : p, pValue->ts);
break;
case TSDB_DATA_TYPE_UTINYINT:
n += tPutU8(p ? p + n : p, pValue->u8);
break;
case TSDB_DATA_TYPE_USMALLINT:
n += tPutU16(p ? p + n : p, pValue->u16);
break;
case TSDB_DATA_TYPE_UINT:
n += tPutU32(p ? p + n : p, pValue->u32);
break;
case TSDB_DATA_TYPE_UBIGINT:
n += tPutU64(p ? p + n : p, pValue->u64);
break;
default:
ASSERT(0);
}
} }
return n;
} }
int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
int32_t n = 0;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL);
} else { } else {
switch (type) { memcpy(&pValue->val, p, tDataTypes[type].bytes);
case TSDB_DATA_TYPE_BOOL: return tDataTypes[type].bytes;
n += tGetI8(p, &pValue->i8);
break;
case TSDB_DATA_TYPE_TINYINT:
n += tGetI8(p, &pValue->i8);
break;
case TSDB_DATA_TYPE_SMALLINT:
n += tGetI16(p, &pValue->i16);
break;
case TSDB_DATA_TYPE_INT:
n += tGetI32(p, &pValue->i32);
break;
case TSDB_DATA_TYPE_BIGINT:
n += tGetI64(p, &pValue->i64);
break;
case TSDB_DATA_TYPE_FLOAT:
n += tGetFloat(p, &pValue->f);
break;
case TSDB_DATA_TYPE_DOUBLE:
n += tGetDouble(p, &pValue->d);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
n += tGetI64(p, &pValue->ts);
break;
case TSDB_DATA_TYPE_UTINYINT:
n += tGetU8(p, &pValue->u8);
break;
case TSDB_DATA_TYPE_USMALLINT:
n += tGetU16(p, &pValue->u16);
break;
case TSDB_DATA_TYPE_UINT:
n += tGetU32(p, &pValue->u32);
break;
case TSDB_DATA_TYPE_UBIGINT:
n += tGetU64(p, &pValue->u64);
break;
default:
ASSERT(0);
}
} }
return n;
} }
int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) {
...@@ -1219,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa ...@@ -1219,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
_exit: _exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag = HAS_VALUE;
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag = HAS_NONE;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag = HAS_NULL;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (COL_VAL_IS_NONE(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
pColData->flag = HAS_NONE; code = tRealloc(&pColData->pBitMap, nBit);
} else if (COL_VAL_IS_NULL(pColVal)) { if (code) return code;
pColData->flag = HAS_NULL;
} else { memset(pColData->pBitMap, 0, nBit);
pColData->flag = HAS_VALUE; SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; pColData->flag |= HAS_VALUE;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) return code;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
}
} }
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (!COL_VAL_IS_NONE(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
code = tRealloc(&pColData->pBitMap, nBit); memset(pColData->pBitMap, 0, nBit);
if (code) goto _exit; SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
memset(pColData->pBitMap, 0, nBit); pColData->flag |= HAS_NULL;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); pColData->nVal++;
if (COL_VAL_IS_NULL(pColVal)) { return code;
pColData->flag |= HAS_NULL; }
} else { static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) {
pColData->flag |= HAS_VALUE; int32_t code = 0;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal); int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
if (code) goto _exit; code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 0, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
pColData->flag |= HAS_VALUE;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) return code;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
} }
} }
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (!COL_VAL_IS_NULL(pColVal)) { int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
int32_t nBit = BIT1_SIZE(pColData->nVal + 1); code = tRealloc(&pColData->pBitMap, nBit);
code = tRealloc(&pColData->pBitMap, nBit); if (code) return code;
if (code) goto _exit;
if (COL_VAL_IS_NONE(pColVal)) { memset(pColData->pBitMap, 255, nBit);
pColData->flag |= HAS_NONE; SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
memset(pColData->pBitMap, 255, nBit); pColData->flag |= HAS_NONE;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0); pColData->nVal++;
} else {
pColData->flag |= HAS_VALUE;
memset(pColData->pBitMap, 0, nBit); return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); }
static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
if (pColData->nVal) { pColData->flag |= HAS_VALUE;
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit;
memset(pColData->pData, 0, pColData->nData);
}
}
code = tColDataPutValue(pColData, pColVal); uint8_t *pBitMap = NULL;
if (code) goto _exit; code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
}
SET_BIT2(pBitMap, pColData->nVal, 2);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
if (pColData->nVal) {
if (IS_VAR_DATA_TYPE(pColData->type)) {
int32_t nOffset = sizeof(int32_t) * pColData->nVal;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) return code;
memset(pColData->aOffset, 0, nOffset);
} else {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) return code;
memset(pColData->pData, 0, pColData->nData);
} }
} }
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (COL_VAL_IS_NONE(pColVal)) { code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); if (code) return code;
if (code) goto _exit;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0); SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else if (COL_VAL_IS_NULL(pColVal)) { pColData->nVal++;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) goto _exit;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); return code;
} else { }
pColData->flag |= HAS_VALUE; static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
uint8_t *pBitMap = NULL; code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code;
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); pColData->nVal++;
}
SET_BIT2(pBitMap, pColData->nVal, 2);
tFree(pColData->pBitMap); return code;
pColData->pBitMap = pBitMap; }
static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
if (pColData->nVal) { code = tColDataPutValue(pColData, pColVal);
if (IS_VAR_DATA_TYPE(pColData->type)) { if (code) return code;
int32_t nOffset = sizeof(int32_t) * pColData->nVal; pColData->nVal++;
code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
if (code) goto _exit; return code;
memset(pColData->aOffset, 0, nOffset); }
} else { static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) {
pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; int32_t code = 0;
code = tRealloc(&pColData->pData, pColData->nData);
if (code) goto _exit; pColData->flag |= HAS_NONE;
memset(pColData->pData, 0, pColData->nData);
} int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
} code = tRealloc(&pColData->pBitMap, nBit);
if (code) return code;
memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (!COL_VAL_IS_VALUE(pColVal)) { pColData->flag |= HAS_NULL;
if (COL_VAL_IS_NONE(pColVal)) {
pColData->flag |= HAS_NONE;
} else {
pColData->flag |= HAS_NULL;
}
int32_t nBit = BIT1_SIZE(pColData->nVal + 1); int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
code = tRealloc(&pColData->pBitMap, nBit); code = tRealloc(&pColData->pBitMap, nBit);
if (code) goto _exit; if (code) return code;
memset(pColData->pBitMap, 255, nBit); memset(pColData->pBitMap, 255, nBit);
SET_BIT1(pColData->pBitMap, pColData->nVal, 0); SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
} else {
code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit;
}
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (COL_VAL_IS_NULL(pColVal)) { code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
pColData->flag |= HAS_NULL; if (code) return code;
uint8_t *pBitMap = NULL; SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { code = tColDataPutValue(pColData, pColVal);
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); if (code) return code;
}
SET_BIT2(pBitMap, pColData->nVal, 1);
tFree(pColData->pBitMap); pColData->nVal++;
pColData->pBitMap = pBitMap;
} else {
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) goto _exit;
if (COL_VAL_IS_NONE(pColVal)) { return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0); }
} else { static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) {
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); int32_t code = 0;
}
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
pColData->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
} }
SET_BIT2(pBitMap, pColData->nVal, 1);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
if (COL_VAL_IS_NONE(pColVal)) { code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
pColData->flag |= HAS_NONE; if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
uint8_t *pBitMap = NULL; code = tColDataPutValue(pColData, pColVal);
code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); if (code) return code;
if (code) goto _exit;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { pColData->nVal++;
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pColData->nVal, 0);
tFree(pColData->pBitMap); return code;
pColData->pBitMap = pBitMap; }
} else { static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) {
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); int32_t code = 0;
if (code) goto _exit;
if (COL_VAL_IS_NULL(pColVal)) { pColData->flag |= HAS_NONE;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
} else { uint8_t *pBitMap = NULL;
SET_BIT1(pColData->pBitMap, pColData->nVal, 1); code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
} if (code) return code;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
} }
SET_BIT2(pBitMap, pColData->nVal, 0);
tFree(pColData->pBitMap);
pColData->pBitMap = pBitMap;
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) {
SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE
int32_t code = 0; int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) goto _exit; if (code) return code;
SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
if (COL_VAL_IS_NONE(pColVal)) {
SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
} else if (COL_VAL_IS_NULL(pColVal)) {
SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
} else {
SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
}
code = tColDataPutValue(pColData, pColVal); code = tColDataPutValue(pColData, pColVal);
if (code) goto _exit; if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++;
return code;
}
static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
if (code) return code;
SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
code = tColDataPutValue(pColData, pColVal);
if (code) return code;
pColData->nVal++; pColData->nVal++;
_exit:
return code; return code;
} }
static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = {
tColDataAppendValue0, // 0 {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0
tColDataAppendValue1, // HAS_NONE {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE
tColDataAppendValue2, // HAS_NULL {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL
tColDataAppendValue3, // HAS_NULL|HAS_NONE {tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE
tColDataAppendValue4, // HAS_VALUE {tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE
tColDataAppendValue5, // HAS_VALUE|HAS_NONE {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE
tColDataAppendValue6, // HAS_VALUE|HAS_NULL {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL
tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
}; };
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal);
} }
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE
......
source/common/src/trow.c
浏览文件 @ 05a39977
...@@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ...@@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
val = varBuf; val = varBuf;
} else { } else {
val = (const void *)&pColVal->value.i64; val = (const void *)&pColVal->value.val;
} }
} else { } else {
pColVal = NULL; pColVal = NULL;
......
source/common/test/dataformatTest.cpp
浏览文件 @ 05a39977
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#if 0
#include <gtest/gtest.h> #include <gtest/gtest.h>
#include <taoserror.h> #include <taoserror.h>
...@@ -476,4 +477,5 @@ TEST(testCase, NoneTest) { ...@@ -476,4 +477,5 @@ TEST(testCase, NoneTest) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
taosMemoryFree(pTSchema); taosMemoryFree(pTSchema);
} }
#endif
#endif #endif
\ No newline at end of file
source/dnode/mnode/impl/src/mndDef.c
浏览文件 @ 05a39977
...@@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { ...@@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; if (pObj->sql != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; } else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
if (pObj->ast != NULL) {
if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1;
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
if (pObj->physicalPlan != NULL) {
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
} else {
if (tEncodeCStr(pEncoder, "") < 0) return -1;
}
int32_t sz = taosArrayGetSize(pObj->tasks); int32_t sz = taosArrayGetSize(pObj->tasks);
if (tEncodeI32(pEncoder, sz) < 0) return -1; if (tEncodeI32(pEncoder, sz) < 0) return -1;
......
source/dnode/mnode/impl/src/mndStb.c
浏览文件 @ 05a39977
...@@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { ...@@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) {
SField *pField1 = taosArrayGet(pCreate->pColumns, i); SField *pField1 = taosArrayGet(pCreate->pColumns, i);
if (pField1->type < 0) { if (pField1->type >= TSDB_DATA_TYPE_MAX) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
...@@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { ...@@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
for (int32_t i = 0; i < pCreate->numOfTags; ++i) { for (int32_t i = 0; i < pCreate->numOfTags; ++i) {
SField *pField1 = taosArrayGet(pCreate->pTags, i); SField *pField1 = taosArrayGet(pCreate->pTags, i);
if (pField1->type < 0) { if (pField1->type >= TSDB_DATA_TYPE_MAX) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
} }
...@@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { ...@@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
} else { } else {
mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name, mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d",
createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer);
terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER;
goto _OVER; goto _OVER;
} }
...@@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa ...@@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
return -1; return -1;
} }
strcpy(pRsp->dbFName, pStb->db); tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName));
strcpy(pRsp->tbName, tbName); tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName));
strcpy(pRsp->stbName, tbName); tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName));
pRsp->dbId = pDb->uid; pRsp->dbId = pDb->uid;
pRsp->numOfTags = pStb->numOfTags; pRsp->numOfTags = pStb->numOfTags;
pRsp->numOfColumns = pStb->numOfColumns; pRsp->numOfColumns = pStb->numOfColumns;
...@@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, ...@@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
return -1; return -1;
} }
strcpy(pRsp->dbFName, pStb->db); tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName));
strcpy(pRsp->tbName, tbName); tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName));
strcpy(pRsp->stbName, tbName); tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName));
pRsp->numOfTags = pStb->numOfTags; pRsp->numOfTags = pStb->numOfTags;
pRsp->numOfColumns = pStb->numOfColumns; pRsp->numOfColumns = pStb->numOfColumns;
pRsp->tableType = TSDB_SUPER_TABLE; pRsp->tableType = TSDB_SUPER_TABLE;
...@@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc ...@@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false);
char rollup[128 + VARSTR_HEADER_SIZE] = {0}; char rollup[160 + VARSTR_HEADER_SIZE] = {0};
int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs);
for (int32_t i = 0; i < rollupNum; ++i) { for (int32_t i = 0; i < rollupNum; ++i) {
char *funcName = taosArrayGet(pStb->pFuncs, i); char *funcName = taosArrayGet(pStb->pFuncs, i);
......
source/dnode/mnode/impl/src/mndStream.c
浏览文件 @ 05a39977
...@@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream ...@@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream
SStreamObj streamObj = {0}; SStreamObj streamObj = {0};
memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN);
streamObj.status = STREAM_STATUS__RECOVER; streamObj.status = STREAM_STATUS__RECOVER;
SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
...@@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { ...@@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1; return -1;
} }
// drop stream // drop stream
if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1; return -1;
} }
...@@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
SName n; SName n;
int32_t cols = 0; int32_t cols = 0;
char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char streamName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName));
tNameGetDbName(&n, varDataVal(streamName));
varDataSetLen(streamName, strlen(varDataVal(streamName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); colDataAppend(pColInfo, numOfRows, (const char *)streamName, false);
...@@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN); STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql));
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)sql, false); colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
char status[20 + VARSTR_HEADER_SIZE] = {0}; char status[20 + VARSTR_HEADER_SIZE] = {0};
mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); char status2[20] = {0};
varDataSetLen(status, strlen(varDataVal(status))); mndShowStreamStatus(status2, pStream);
STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&status, false); colDataAppend(pColInfo, numOfRows, (const char *)&status, false);
char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB));
tNameGetDbName(&n, varDataVal(sourceDB));
varDataSetLen(sourceDB, strlen(varDataVal(sourceDB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false);
char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB));
tNameGetDbName(&n, varDataVal(targetDB));
varDataSetLen(targetDB, strlen(varDataVal(targetDB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false);
...@@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, NULL, true); colDataAppend(pColInfo, numOfRows, NULL, true);
} else { } else {
char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB));
strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n));
varDataSetLen(targetSTB, strlen(varDataVal(targetSTB)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false);
} }
...@@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB ...@@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false);
char trigger[20 + VARSTR_HEADER_SIZE] = {0}; char trigger[20 + VARSTR_HEADER_SIZE] = {0};
mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); char trigger2[20] = {0};
varDataSetLen(trigger, strlen(varDataVal(trigger))); mndShowStreamTrigger(trigger2, pStream);
STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false);
......
source/dnode/vnode/inc/vnode.h
浏览文件 @ 05a39977
...@@ -96,7 +96,7 @@ void metaReaderClear(SMetaReader *pReader); ...@@ -96,7 +96,7 @@ void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags);
int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaGetTableTagsByUids(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags);
int32_t metaReadNext(SMetaReader *pReader); int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
...@@ -159,7 +159,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader); ...@@ -159,7 +159,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids);
void* tsdbCacherowsReaderClose(void *pReader); void *tsdbCacherowsReaderClose(void *pReader);
int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
......
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @ 05a39977
...@@ -38,39 +38,43 @@ extern "C" { ...@@ -38,39 +38,43 @@ extern "C" {
goto LABEL; \ goto LABEL; \
} }
typedef struct TSDBROW TSDBROW; typedef struct TSDBROW TSDBROW;
typedef struct TABLEID TABLEID; typedef struct TABLEID TABLEID;
typedef struct TSDBKEY TSDBKEY; typedef struct TSDBKEY TSDBKEY;
typedef struct SDelData SDelData; typedef struct SDelData SDelData;
typedef struct SDelIdx SDelIdx; typedef struct SDelIdx SDelIdx;
typedef struct STbData STbData; typedef struct STbData STbData;
typedef struct SMemTable SMemTable; typedef struct SMemTable SMemTable;
typedef struct STbDataIter STbDataIter; typedef struct STbDataIter STbDataIter;
typedef struct SMapData SMapData; typedef struct SMapData SMapData;
typedef struct SBlockIdx SBlockIdx; typedef struct SBlockIdx SBlockIdx;
typedef struct SDataBlk SDataBlk; typedef struct SDataBlk SDataBlk;
typedef struct SSttBlk SSttBlk; typedef struct SSttBlk SSttBlk;
typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SDiskDataHdr SDiskDataHdr;
typedef struct SBlockData SBlockData; typedef struct SBlockData SBlockData;
typedef struct SDelFile SDelFile; typedef struct SDelFile SDelFile;
typedef struct SHeadFile SHeadFile; typedef struct SHeadFile SHeadFile;
typedef struct SDataFile SDataFile; typedef struct SDataFile SDataFile;
typedef struct SSttFile SSttFile; typedef struct SSttFile SSttFile;
typedef struct SSmaFile SSmaFile; typedef struct SSmaFile SSmaFile;
typedef struct SDFileSet SDFileSet; typedef struct SDFileSet SDFileSet;
typedef struct SDataFWriter SDataFWriter; typedef struct SDataFWriter SDataFWriter;
typedef struct SDataFReader SDataFReader; typedef struct SDataFReader SDataFReader;
typedef struct SDelFWriter SDelFWriter; typedef struct SDelFWriter SDelFWriter;
typedef struct SDelFReader SDelFReader; typedef struct SDelFReader SDelFReader;
typedef struct SRowIter SRowIter; typedef struct SRowIter SRowIter;
typedef struct STsdbFS STsdbFS; typedef struct STsdbFS STsdbFS;
typedef struct SRowMerger SRowMerger; typedef struct SRowMerger SRowMerger;
typedef struct STsdbReadSnap STsdbReadSnap; typedef struct STsdbReadSnap STsdbReadSnap;
typedef struct SBlockInfo SBlockInfo; typedef struct SBlockInfo SBlockInfo;
typedef struct SSmaInfo SSmaInfo; typedef struct SSmaInfo SSmaInfo;
typedef struct SBlockCol SBlockCol; typedef struct SBlockCol SBlockCol;
typedef struct SVersionRange SVersionRange; typedef struct SVersionRange SVersionRange;
typedef struct SLDataIter SLDataIter; typedef struct SLDataIter SLDataIter;
typedef struct SDiskCol SDiskCol;
typedef struct SDiskData SDiskData;
typedef struct SDiskDataBuilder SDiskDataBuilder;
typedef struct SBlkInfo SBlkInfo;
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8 #define TSDB_MAX_SUBBLOCKS 8
...@@ -170,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut ...@@ -170,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut
int32_t aBufN[]); int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
// SDiskDataHdr // SDiskDataHdr
int32_t tPutDiskDataHdr(uint8_t *p, void *ph); int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr);
int32_t tGetDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
// SDelIdx // SDelIdx
int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tPutDelIdx(uint8_t *p, void *ph);
...@@ -267,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p ...@@ -267,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast); int8_t cmprAlg, int8_t toLast);
int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo);
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
// SDataFReader // SDataFReader
...@@ -300,7 +305,7 @@ int32_t tsdbMerge(STsdb *pTsdb); ...@@ -300,7 +305,7 @@ int32_t tsdbMerge(STsdb *pTsdb);
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) #define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0)
// tsdbCache // tsdbCache ==============================================================================================
int32_t tsdbOpenCache(STsdb *pTsdb); int32_t tsdbOpenCache(STsdb *pTsdb);
void tsdbCloseCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb);
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
...@@ -318,6 +323,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); ...@@ -318,6 +323,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode);
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
// tsdbDiskData ==============================================================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder);
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
uint8_t calcSma);
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
// structs ======================= // structs =======================
struct STsdbFS { struct STsdbFS {
SDelFile *pDelFile; SDelFile *pDelFile;
...@@ -438,6 +452,17 @@ struct SSmaInfo { ...@@ -438,6 +452,17 @@ struct SSmaInfo {
int32_t size; int32_t size;
}; };
struct SBlkInfo {
int64_t minUid;
int64_t maxUid;
TSKEY minKey;
TSKEY maxKey;
int64_t minVer;
int64_t maxVer;
TSDBKEY minTKey;
TSDBKEY maxTKey;
};
struct SDataBlk { struct SDataBlk {
TSDBKEY minKey; TSDBKEY minKey;
TSDBKEY maxKey; TSDBKEY maxKey;
...@@ -661,6 +686,38 @@ typedef struct { ...@@ -661,6 +686,38 @@ typedef struct {
STSchema *pTSchema; STSchema *pTSchema;
} SSkmInfo; } SSkmInfo;
struct SDiskCol {
SBlockCol bCol;
const uint8_t *pBit;
const uint8_t *pOff;
const uint8_t *pVal;
SColumnDataAgg agg;
};
struct SDiskData {
SDiskDataHdr hdr;
const uint8_t *pUid;
const uint8_t *pVer;
const uint8_t *pKey;
SArray *aDiskCol; // SArray<SDiskCol>
};
struct SDiskDataBuilder {
int64_t suid;
int64_t uid;
int32_t nRow;
uint8_t cmprAlg;
uint8_t calcSma;
SCompressor *pUidC;
SCompressor *pVerC;
SCompressor *pKeyC;
int32_t nBuilder;
SArray *aBuilder; // SArray<SDiskColBuilder>
uint8_t *aBuf[2];
SDiskData dd;
SBlkInfo bi;
};
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr); bool destroyLoadInfo, const char *idStr);
......
source/dnode/vnode/src/meta/metaQuery.c
浏览文件 @ 05a39977
...@@ -207,17 +207,17 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) { ...@@ -207,17 +207,17 @@ int metaGetTableUidByName(void *meta, char *tbName, uint64_t *uid) {
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, (SMeta *)meta, 0); metaReaderInit(&mr, (SMeta *)meta, 0);
SMeta *pMeta = mr.pMeta;
SMetaReader *pReader = &mr; SMetaReader *pReader = &mr;
// query name.idx // query name.idx
if (tdbTbGet(pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) { if (tdbTbGet(pReader->pMeta->pNameIdx, tbName, strlen(tbName) + 1, &pReader->pBuf, &pReader->szBuf) < 0) {
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
metaReaderClear(&mr); metaReaderClear(&mr);
return -1; return -1;
} }
*uid = *(tb_uid_t *)pReader->pBuf; *uid = *(tb_uid_t *)pReader->pBuf;
metaReaderClear(&mr); metaReaderClear(&mr);
return 0; return 0;
...@@ -228,11 +228,11 @@ int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) { ...@@ -228,11 +228,11 @@ int metaGetTableTypeByName(void *meta, char *tbName, ETableType *tbType) {
SMetaReader mr = {0}; SMetaReader mr = {0};
metaReaderInit(&mr, (SMeta *)meta, 0); metaReaderInit(&mr, (SMeta *)meta, 0);
if (metaGetTableEntryByName(&mr, tbName) == 0) { code = metaGetTableEntryByName(&mr, tbName);
*tbType = mr.me.type; if (code == 0) *tbType = mr.me.type;
}
metaReaderClear(&mr); metaReaderClear(&mr);
return 0; return code;
} }
int metaReadNext(SMetaReader *pReader) { int metaReadNext(SMetaReader *pReader) {
...@@ -1134,21 +1134,49 @@ END: ...@@ -1134,21 +1134,49 @@ END:
return ret; return ret;
} }
int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { static int32_t metaGetTableTagByUid(SMeta *pMeta, uint64_t suid, uint64_t uid, void **tag, int32_t *len, bool lock) {
int ret = 0;
if (lock) {
metaRLock(pMeta);
}
SCtbIdxKey ctbIdxKey = {.suid = suid, .uid = uid};
ret = tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), tag, len);
if (lock) {
metaULock(pMeta);
}
return ret;
}
int32_t metaGetTableTagsByUids(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) {
const int32_t LIMIT = 128;
int32_t isLock = false;
int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; int32_t sz = uidList ? taosArrayGetSize(uidList) : 0;
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
tb_uid_t *id = taosArrayGet(uidList, i); tb_uid_t *id = taosArrayGet(uidList, i);
SCtbIdxKey ctbIdxKey = {.suid = suid, .uid = *id};
if (i % LIMIT == 0) {
void *val = NULL; if (isLock) metaULock(pMeta);
int32_t len = 0;
if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL && metaRLock(pMeta);
0 == tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), &val, &len)) { isLock = true;
taosHashPut(tags, id, sizeof(tb_uid_t), val, len); }
if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL) {
void *val = NULL;
int32_t len = 0;
if (metaGetTableTagByUid(pMeta, suid, *id, &val, &len, false) == 0) {
taosHashPut(tags, id, sizeof(tb_uid_t), val, len);
tdbFree(val);
}
} }
} }
if (isLock) metaULock(pMeta);
return 0; return 0;
} }
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) {
SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid);
......
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @ 05a39977
...@@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb ...@@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
if (keyTs > tTsVal->ts) { if (keyTs > tTsVal->ts) {
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs}); SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs});
taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal});
} }
...@@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { ...@@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
if (--state->iFileSet >= 0) { if (--state->iFileSet >= 0) {
pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet);
} else { } else {
// tMergeTreeClose(&state->mergeTree); tMergeTreeClose(&state->mergeTree);
*ppRow = NULL; *ppRow = NULL;
return code; return code;
...@@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { ...@@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) {
bool hasVal = tMergeTreeNext(&state->mergeTree); bool hasVal = tMergeTreeNext(&state->mergeTree);
if (!hasVal) { if (!hasVal) {
state->state = SFSLASTNEXTROW_FILESET; state->state = SFSLASTNEXTROW_FILESET;
// tMergeTreeClose(&state->mergeTree); tMergeTreeClose(&state->mergeTree);
goto _next_fileset; goto _next_fileset;
} }
state->state = SFSLASTNEXTROW_BLOCKROW; state->state = SFSLASTNEXTROW_BLOCKROW;
...@@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { ...@@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) {
goto _next_fileset; goto _next_fileset;
} }
tMapDataReset(&state->blockMap); if (state->blockMap.pData != NULL) {
tMapDataClear(&state->blockMap);
}
code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap);
if (code) goto _err; if (code) goto _err;
...@@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) { ...@@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) {
state->pBlockData = NULL; state->pBlockData = NULL;
} }
if (state->blockMap.pData != NULL) {
tMapDataClear(&state->blockMap);
}
return code; return code;
} }
...@@ -1052,7 +1059,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo ...@@ -1052,7 +1059,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
lastRowTs = TSDBROW_TS(pRow); lastRowTs = TSDBROW_TS(pRow);
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, pColVal) == NULL) { if (taosArrayPush(pColArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
...@@ -1151,7 +1158,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { ...@@ -1151,7 +1158,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
lastRowTs = rowTs; lastRowTs = rowTs;
STColumn *pTColumn = &pTSchema->columns[0]; STColumn *pTColumn = &pTSchema->columns[0];
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
......
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @ 05a39977
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT;
#define USE_STREAM_COMPRESSION 0
typedef struct { typedef struct {
SRBTreeNode n; SRBTreeNode n;
SRowInfo r; SRowInfo r;
...@@ -75,7 +77,11 @@ typedef struct { ...@@ -75,7 +77,11 @@ typedef struct {
SArray *aSttBlk; // SArray<SSttBlk> SArray *aSttBlk; // SArray<SSttBlk>
SMapData mBlock; // SMapData<SDataBlk> SMapData mBlock; // SMapData<SDataBlk>
SBlockData bData; SBlockData bData;
SBlockData bDatal; #if USE_STREAM_COMPRESSION
SDiskDataBuilder *pBuilder;
#else
SBlockData bDatal;
#endif
} dWriter; } dWriter;
SSkmInfo skmTable; SSkmInfo skmTable;
SSkmInfo skmRow; SSkmInfo skmRow;
...@@ -115,33 +121,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) { ...@@ -115,33 +121,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
int32_t tsdbBegin(STsdb *pTsdb) { int32_t tsdbBegin(STsdb *pTsdb) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
if (!pTsdb) return code; if (!pTsdb) return code;
SMemTable *pMemTable; SMemTable *pMemTable;
code = tsdbMemTableCreate(pTsdb, &pMemTable); code = tsdbMemTableCreate(pTsdb, &pMemTable);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// lock // lock
code = taosThreadRwlockWrlock(&pTsdb->rwLock); if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) {
if (code) {
code = TAOS_SYSTEM_ERROR(code); code = TAOS_SYSTEM_ERROR(code);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
pTsdb->mem = pMemTable; pTsdb->mem = pMemTable;
// unlock // unlock
code = taosThreadRwlockUnlock(&pTsdb->rwLock); if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) {
if (code) {
code = TAOS_SYSTEM_ERROR(code); code = TAOS_SYSTEM_ERROR(code);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbError("vgId:%d, tsdb begin failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
...@@ -149,6 +154,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -149,6 +154,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
if (!pTsdb) return 0; if (!pTsdb) return 0;
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SCommitter commith; SCommitter commith;
SMemTable *pMemTable = pTsdb->mem; SMemTable *pMemTable = pTsdb->mem;
...@@ -164,76 +170,74 @@ int32_t tsdbCommit(STsdb *pTsdb) { ...@@ -164,76 +170,74 @@ int32_t tsdbCommit(STsdb *pTsdb) {
// start commit // start commit
code = tsdbStartCommit(pTsdb, &commith); code = tsdbStartCommit(pTsdb, &commith);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// commit impl // commit impl
code = tsdbCommitData(&commith); code = tsdbCommitData(&commith);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitDel(&commith); code = tsdbCommitDel(&commith);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// end commit // end commit
code = tsdbEndCommit(&commith, 0); code = tsdbEndCommit(&commith, 0);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
return code; if (code) {
tsdbEndCommit(&commith, code);
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbEndCommit(&commith, code); }
tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
if (pCommitter->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData)); if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
if (pCommitter->aDelData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx)); if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
if (pCommitter->aDelIdxN == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
SDelFile *pDelFileR = pCommitter->fs.pDelFile; SDelFile *pDelFileR = pCommitter->fs.pDelFile;
if (pDelFileR) { if (pDelFileR) {
code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
// prepare new // prepare new
SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0};
code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); if (code) {
return code; tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
} else {
_err: tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode));
tsdbError("vgId:%d, commit del start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SDelData *pDelData; SDelData *pDelData;
tb_uid_t suid; tb_uid_t suid;
tb_uid_t uid; tb_uid_t uid;
...@@ -252,7 +256,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel ...@@ -252,7 +256,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
uid = pDelIdx->uid; uid = pDelIdx->uid;
code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
taosArrayClear(pCommitter->aDelData); taosArrayClear(pCommitter->aDelData);
} }
...@@ -266,62 +270,65 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel ...@@ -266,62 +270,65 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
for (; pDelData; pDelData = pDelData->pNext) { for (; pDelData; pDelData = pDelData->pNext) {
if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
// write // write
code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// put delIdx // put delIdx
if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
return code; if (code) {
tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
_err: tstrerror(code));
tsdbError("vgId:%d, commit table del failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->pDelFReader) { if (pCommitter->pDelFReader) {
code = tsdbDelFReaderClose(&pCommitter->pDelFReader); code = tsdbDelFReaderClose(&pCommitter->pDelFReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
taosArrayDestroy(pCommitter->aDelIdx); taosArrayDestroy(pCommitter->aDelIdx);
taosArrayDestroy(pCommitter->aDelData); taosArrayDestroy(pCommitter->aDelData);
taosArrayDestroy(pCommitter->aDelIdxN); taosArrayDestroy(pCommitter->aDelIdxN);
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tsdbError("vgId:%d, commit del end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tstrerror(code));
}
return code; return code;
} }
int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
if (suid) { if (suid) {
if (pSkmInfo->suid == suid) { if (pSkmInfo->suid == suid) {
...@@ -336,7 +343,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo ...@@ -336,7 +343,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo
pSkmInfo->uid = uid; pSkmInfo->uid = uid;
tTSchemaDestroy(pSkmInfo->pTSchema); tTSchemaDestroy(pSkmInfo->pTSchema);
code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
return code; return code;
...@@ -344,6 +351,7 @@ _exit: ...@@ -344,6 +351,7 @@ _exit:
static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
if (pCommitter->skmRow.pTSchema) { if (pCommitter->skmRow.pTSchema) {
if (pCommitter->skmRow.suid == suid) { if (pCommitter->skmRow.suid == suid) {
...@@ -359,9 +367,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid ...@@ -359,9 +367,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid
pCommitter->skmRow.uid = uid; pCommitter->skmRow.uid = uid;
tTSchemaDestroy(pCommitter->skmRow.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema);
code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _exit;
}
_exit: _exit:
return code; return code;
...@@ -369,6 +375,7 @@ _exit: ...@@ -369,6 +375,7 @@ _exit:
static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
ASSERT(pCommitter->dReader.pBlockIdx); ASSERT(pCommitter->dReader.pBlockIdx);
...@@ -378,7 +385,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { ...@@ -378,7 +385,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(pCommitter->dReader.mBlock.nItem > 0); ASSERT(pCommitter->dReader.mBlock.nItem > 0);
} else { } else {
...@@ -391,6 +398,7 @@ _exit: ...@@ -391,6 +398,7 @@ _exit:
static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
pCommitter->pIter = NULL; pCommitter->pIter = NULL;
tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn);
...@@ -431,14 +439,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { ...@@ -431,14 +439,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
pIter->iStt = iStt; pIter->iStt = iStt;
code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; if (taosArrayGetSize(pIter->aSttBlk) == 0) continue;
pIter->iSttBlk = 0; pIter->iSttBlk = 0;
SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0);
code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pIter->iRow = 0; pIter->iRow = 0;
pIter->r.suid = pIter->bData.suid; pIter->r.suid = pIter->bData.suid;
...@@ -460,16 +468,19 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { ...@@ -460,16 +468,19 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
} }
code = tsdbNextCommitRow(pCommitter); code = tsdbNextCommitRow(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code;
_err: _exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SDFileSet *pRSet = NULL; SDFileSet *pRSet = NULL;
...@@ -484,17 +495,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -484,17 +495,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ);
if (pRSet) { if (pRSet) {
code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// data // data
code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pCommitter->dReader.iBlockIdx = 0; pCommitter->dReader.iBlockIdx = 0;
if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) {
pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0);
code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} else { } else {
pCommitter->dReader.pBlockIdx = NULL; pCommitter->dReader.pBlockIdx = NULL;
} }
...@@ -531,28 +542,32 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { ...@@ -531,28 +542,32 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
} }
wSet.aSttF[wSet.nSttF - 1] = &fStt; wSet.aSttF[wSet.nSttF - 1] = &fStt;
code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aBlockIdx);
taosArrayClear(pCommitter->dWriter.aSttBlk); taosArrayClear(pCommitter->dWriter.aSttBlk);
tMapDataReset(&pCommitter->dWriter.mBlock); tMapDataReset(&pCommitter->dWriter.mBlock);
tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bData);
#if USE_STREAM_COMPRESSION
tDiskDataBuilderClear(pCommitter->dWriter.pBuilder);
#else
tBlockDataReset(&pCommitter->dWriter.bDatal); tBlockDataReset(&pCommitter->dWriter.bDatal);
#endif
// open iter // open iter
code = tsdbOpenCommitIter(pCommitter); code = tsdbOpenCommitIter(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
return code; if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
_err: }
tsdbError("vgId:%d, commit file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
if (pBlockData->nRow == 0) return code; if (pBlockData->nRow == 0) return code;
...@@ -586,24 +601,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa ...@@ -586,24 +601,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa
dataBlk.nSubBlock++; dataBlk.nSubBlock++;
code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1],
((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// put SDataBlk // put SDataBlk
code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// clear // clear
tBlockDataClear(pBlockData); tBlockDataClear(pBlockData);
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SSttBlk sstBlk; SSttBlk sstBlk;
if (pBlockData->nRow == 0) return code; if (pBlockData->nRow == 0) return code;
...@@ -626,114 +642,159 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray ...@@ -626,114 +642,159 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray
// write // write
code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// push SSttBlk // push SSttBlk
if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { if (taosArrayPush(aSttBlk, &sstBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
// clear // clear
tBlockDataClear(pBlockData); tBlockDataClear(pBlockData);
_exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code; return code;
}
static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) {
int32_t code = 0;
int32_t lino = 0;
if (pBuilder->nRow == 0) return code;
// gnrt
const SDiskData *pDiskData;
const SBlkInfo *pBlkInfo;
code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo);
TSDB_CHECK_CODE(code, lino, _exit);
SSttBlk sttBlk = {.suid = pBuilder->suid,
.minUid = pBlkInfo->minUid,
.maxUid = pBlkInfo->maxUid,
.minKey = pBlkInfo->minKey,
.maxKey = pBlkInfo->maxKey,
.minVer = pBlkInfo->minVer,
.maxVer = pBlkInfo->maxVer,
.nRow = pBuilder->nRow};
// write
code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL);
TSDB_CHECK_CODE(code, lino, _exit);
// push
if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
// clear
tDiskDataBuilderClear(pBuilder);
_err: _exit:
tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
// write aBlockIdx // write aBlockIdx
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// write aSttBlk // write aSttBlk
code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// update file header // update file header
code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// upsert SDFileSet // upsert SDFileSet
code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// close and sync // close and sync
code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->dReader.pReader) { if (pCommitter->dReader.pReader) {
code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); code = tsdbDataFReaderClose(&pCommitter->dReader.pReader);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
return code; if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
_err: tstrerror(code));
tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) {
SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbCommitterNextTableData(pCommitter); code = tsdbCommitterNextTableData(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter);
static int32_t tsdbCommitFileData(SCommitter *pCommitter) { static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
// commit file data start // commit file data start
code = tsdbCommitFileDataStart(pCommitter); code = tsdbCommitFileDataStart(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// impl // impl
code = tsdbCommitFileDataImpl(pCommitter); code = tsdbCommitFileDataImpl(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// commit file data end // commit file data end
code = tsdbCommitFileDataEnd(pCommitter); code = tsdbCommitFileDataEnd(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code;
_err: _exit:
tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); if (code) {
tsdbDataFReaderClose(&pCommitter->dReader.pReader); tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); tsdbDataFReaderClose(&pCommitter->dReader.pReader);
tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0);
}
return code; return code;
} }
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
memset(pCommitter, 0, sizeof(*pCommitter)); memset(pCommitter, 0, sizeof(*pCommitter));
ASSERT(pTsdb->mem && pTsdb->imem == NULL); ASSERT(pTsdb->mem && pTsdb->imem == NULL);
...@@ -754,30 +815,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { ...@@ -754,30 +815,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) { if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tsdbFSCopy(pTsdb, &pCommitter->fs); code = tsdbFSCopy(pTsdb, &pCommitter->fs);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
// reader // reader
pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dReader.aBlockIdx == NULL) { if (pCommitter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tBlockDataCreate(&pCommitter->dReader.bData); code = tBlockDataCreate(&pCommitter->dReader.bData);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
// merger // merger
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
...@@ -785,33 +847,41 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { ...@@ -785,33 +847,41 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) { if (pIter->aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tBlockDataCreate(&pIter->bData); code = tBlockDataCreate(&pIter->bData);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
// writer // writer
pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dWriter.aBlockIdx == NULL) { if (pCommitter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pCommitter->dWriter.aSttBlk == NULL) { if (pCommitter->dWriter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
code = tBlockDataCreate(&pCommitter->dWriter.bData); code = tBlockDataCreate(&pCommitter->dWriter.bData);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
#if USE_STREAM_COMPRESSION
code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder);
#else
code = tBlockDataCreate(&pCommitter->dWriter.bDatal); code = tBlockDataCreate(&pCommitter->dWriter.bDatal);
if (code) goto _exit; #endif
TSDB_CHECK_CODE(code, lino, _exit);
_exit: _exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
...@@ -833,13 +903,19 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { ...@@ -833,13 +903,19 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
taosArrayDestroy(pCommitter->dWriter.aSttBlk); taosArrayDestroy(pCommitter->dWriter.aSttBlk);
tMapDataClear(&pCommitter->dWriter.mBlock); tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1); tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
#if USE_STREAM_COMPRESSION
tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder);
#else
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
#endif
tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmTable.pTSchema);
tTSchemaDestroy(pCommitter->skmRow.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema);
} }
static int32_t tsdbCommitData(SCommitter *pCommitter) { static int32_t tsdbCommitData(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
...@@ -848,30 +924,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { ...@@ -848,30 +924,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) {
// start ==================== // start ====================
code = tsdbCommitDataStart(pCommitter); code = tsdbCommitDataStart(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// impl ==================== // impl ====================
pCommitter->nextKey = pMemTable->minKey; pCommitter->nextKey = pMemTable->minKey;
while (pCommitter->nextKey < TSKEY_MAX) { while (pCommitter->nextKey < TSKEY_MAX) {
code = tsdbCommitFileData(pCommitter); code = tsdbCommitFileData(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
// end ==================== // end ====================
tsdbCommitDataEnd(pCommitter); tsdbCommitDataEnd(pCommitter);
_exit: _exit:
tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); if (code) {
return code; tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
}
_err:
tsdbCommitDataEnd(pCommitter);
tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
static int32_t tsdbCommitDel(SCommitter *pCommitter) { static int32_t tsdbCommitDel(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
...@@ -882,7 +957,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { ...@@ -882,7 +957,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
// start // start
code = tsdbCommitDelStart(pCommitter); code = tsdbCommitDelStart(pCommitter);
if (code) { if (code) {
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
// impl // impl
...@@ -918,7 +993,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { ...@@ -918,7 +993,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_mem_del: _commit_mem_del:
code = tsdbCommitTableDel(pCommitter, pTbData, NULL); code = tsdbCommitTableDel(pCommitter, pTbData, NULL);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iTbData++; iTbData++;
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
...@@ -926,7 +1001,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { ...@@ -926,7 +1001,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_disk_del: _commit_disk_del:
code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iDelIdx++; iDelIdx++;
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
...@@ -934,7 +1009,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { ...@@ -934,7 +1009,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_mem_and_disk_del: _commit_mem_and_disk_del:
code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iTbData++; iTbData++;
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
...@@ -945,28 +1020,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { ...@@ -945,28 +1020,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
// end // end
code = tsdbCommitDelEnd(pCommitter); code = tsdbCommitDelEnd(pCommitter);
if (code) { TSDB_CHECK_CODE(code, lino, _exit);
goto _err;
}
_exit: _exit:
tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); if (code) {
return code; tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
} else {
_err: tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb; STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem; SMemTable *pMemTable = pTsdb->imem;
ASSERT(eno == 0); ASSERT(eno == 0);
code = tsdbFSCommit1(pTsdb, &pCommitter->fs); code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// lock // lock
taosThreadRwlockWrlock(&pTsdb->rwLock); taosThreadRwlockWrlock(&pTsdb->rwLock);
...@@ -975,7 +1050,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { ...@@ -975,7 +1050,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
code = tsdbFSCommit2(pTsdb, &pCommitter->fs); code = tsdbFSCommit2(pTsdb, &pCommitter->fs);
if (code) { if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock); taosThreadRwlockUnlock(&pTsdb->rwLock);
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
pTsdb->imem = NULL; pTsdb->imem = NULL;
...@@ -987,16 +1062,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { ...@@ -987,16 +1062,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
tsdbFSDestroy(&pCommitter->fs); tsdbFSDestroy(&pCommitter->fs);
taosArrayDestroy(pCommitter->aTbDataP); taosArrayDestroy(pCommitter->aTbDataP);
// if (pCommitter->toMerge) { _exit:
// code = tsdbMerge(pTsdb); if (code) {
// if (code) goto _err; tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
// } } else {
tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode));
tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); }
return code;
_err:
tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
return code; return code;
} }
...@@ -1008,6 +1079,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { ...@@ -1008,6 +1079,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) {
static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
if (pCommitter->pIter) { if (pCommitter->pIter) {
SDataIter *pIter = pCommitter->pIter; SDataIter *pIter = pCommitter->pIter;
...@@ -1085,11 +1157,17 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { ...@@ -1085,11 +1157,17 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
} }
_exit: _exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SBlockData *pBlockData = &pCommitter->dWriter.bData; SBlockData *pBlockData = &pCommitter->dWriter.bData;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
...@@ -1098,13 +1176,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1098,13 +1176,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
while (pRowInfo) { while (pRowInfo) {
ASSERT(pRowInfo->row.type == 0); ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter); code = tsdbNextCommitRow(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo) { if (pRowInfo) {
...@@ -1119,29 +1197,32 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1119,29 +1197,32 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBlockData->nRow >= pCommitter->maxRow) { if (pBlockData->nRow >= pCommitter->maxRow) {
code = code =
tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code;
_err: _exit:
tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataR = &pCommitter->dReader.bData;
SBlockData *pBDataW = &pCommitter->dWriter.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData;
code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
tBlockDataClear(pBDataW); tBlockDataClear(pBDataW);
int32_t iRow = 0; int32_t iRow = 0;
...@@ -1152,7 +1233,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1152,7 +1233,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row);
if (c < 0) { if (c < 0) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iRow++; iRow++;
if (iRow < pBDataR->nRow) { if (iRow < pBDataR->nRow) {
...@@ -1163,13 +1244,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1163,13 +1244,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
} else if (c > 0) { } else if (c > 0) {
ASSERT(pRowInfo->row.type == 0); ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter); code = tsdbNextCommitRow(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo) { if (pRowInfo) {
...@@ -1186,13 +1267,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1186,13 +1267,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBDataW->nRow >= pCommitter->maxRow) { if (pBDataW->nRow >= pCommitter->maxRow) {
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
while (pRow) { while (pRow) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iRow++; iRow++;
if (iRow < pBDataR->nRow) { if (iRow < pBDataR->nRow) {
...@@ -1203,22 +1284,25 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) ...@@ -1203,22 +1284,25 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBDataW->nRow >= pCommitter->maxRow) { if (pBDataW->nRow >= pCommitter->maxRow) {
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
return code;
_err: _exit:
tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx;
ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0);
...@@ -1237,7 +1321,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { ...@@ -1237,7 +1321,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
if (c < 0) { if (c < 0) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
...@@ -1247,13 +1331,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { ...@@ -1247,13 +1331,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
} }
} else if (c > 0) { } else if (c > 0) {
code = tsdbCommitAheadBlock(pCommitter, pDataBlk); code = tsdbCommitAheadBlock(pCommitter, pDataBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
} else { } else {
code = tsdbCommitMergeBlock(pCommitter, pDataBlk); code = tsdbCommitMergeBlock(pCommitter, pDataBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
...@@ -1268,7 +1352,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { ...@@ -1268,7 +1352,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
while (pDataBlk) { while (pDataBlk) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
iBlock++; iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) { if (iBlock < pCommitter->dReader.mBlock.nItem) {
...@@ -1279,70 +1363,113 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { ...@@ -1279,70 +1363,113 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
} }
code = tsdbCommitterNextTableData(pCommitter); code = tsdbCommitterNextTableData(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
_exit: _exit:
return code; if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
_err: tstrerror(code));
tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
#if USE_STREAM_COMPRESSION
SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder;
if (pBuilder->suid || pBuilder->uid) {
if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) {
code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
tDiskDataBuilderClear(pBuilder);
}
}
SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (!pBuilder->suid && !pBuilder->uid) {
if (pBDatal->suid || pBDatal->uid) { ASSERT(pCommitter->skmTable.suid == id.suid);
if ((pBDatal->suid != id.suid) || (id.suid == 0)) { ASSERT(pCommitter->skmTable.uid == id.uid);
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
tBlockDataReset(pBDatal); }
#else
SBlockData *pBData = &pCommitter->dWriter.bDatal;
if (pBData->suid || pBData->uid) {
if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) {
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
tBlockDataReset(pBData);
} }
} }
if (!pBDatal->suid && !pBDatal->uid) { if (!pBData->suid && !pBData->uid) {
ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.suid == id.suid);
ASSERT(pCommitter->skmTable.uid == id.uid); ASSERT(pCommitter->skmTable.uid == id.uid);
TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid};
code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0); code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _exit; TSDB_CHECK_CODE(code, lino, _exit);
} }
#endif
_exit: _exit:
if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SBlockData *pBData = &pCommitter->dWriter.bData; SBlockData *pBData = &pCommitter->dWriter.bData;
SBlockData *pBDatal = &pCommitter->dWriter.bDatal; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid};
TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
code = tsdbInitLastBlockIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit);
if (code) goto _err;
for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) {
TSDBROW row = tsdbRowFromBlockData(pBData, iRow); TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid);
if (code) goto _err;
if (pBDatal->nRow >= pCommitter->maxRow) { #if USE_STREAM_COMPRESSION
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) {
code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
TSDB_CHECK_CODE(code, lino, _exit);
}
#else
code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid);
TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) {
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
pCommitter->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
} }
#endif
} }
return code; _exit:
if (code) {
_err: tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
...@@ -1351,67 +1478,102 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { ...@@ -1351,67 +1478,102 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
if (pRowInfo == NULL) goto _exit; if (pRowInfo == NULL) goto _exit;
SBlockData *pBData;
if (pCommitter->toLastOnly) { if (pCommitter->toLastOnly) {
pBData = &pCommitter->dWriter.bDatal; code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
code = tsdbInitLastBlockIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit);
if (code) goto _err;
while (pRowInfo) {
STSchema *pTSchema = NULL;
if (pRowInfo->row.type == 0) {
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
TSDB_CHECK_CODE(code, lino, _exit);
pTSchema = pCommitter->skmRow.pTSchema;
}
#if USE_STREAM_COMPRESSION
code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id);
#else
code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid);
#endif
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter);
TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
pRowInfo = NULL;
}
#if USE_STREAM_COMPRESSION
if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) {
code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
TSDB_CHECK_CODE(code, lino, _exit);
}
#else
if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow) {
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
pCommitter->cmprAlg);
TSDB_CHECK_CODE(code, lino, _exit);
}
#endif
}
} else { } else {
pBData = &pCommitter->dWriter.bData; SBlockData *pBData = &pCommitter->dWriter.bData;
ASSERT(pBData->nRow == 0); ASSERT(pBData->nRow == 0);
}
while (pRowInfo) { while (pRowInfo) {
STSchema *pTSchema = NULL; STSchema *pTSchema = NULL;
if (pRowInfo->row.type == 0) { if (pRowInfo->row.type == 0) {
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pTSchema = pCommitter->skmRow.pTSchema; pTSchema = pCommitter->skmRow.pTSchema;
} }
code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter); code = tsdbNextCommitRow(pCommitter);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter); pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
pRowInfo = NULL; pRowInfo = NULL;
} }
if (pBData->nRow >= pCommitter->maxRow) { if (pBData->nRow >= pCommitter->maxRow) {
if (pCommitter->toLastOnly) {
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
if (code) goto _err;
} else {
code = code =
tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
}
if (!pCommitter->toLastOnly && pBData->nRow) { if (pBData->nRow) {
if (pBData->nRow > pCommitter->minRow) { if (pBData->nRow > pCommitter->minRow) {
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); code =
if (code) goto _err; tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
} else { TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbAppendLastBlock(pCommitter); } else {
if (code) goto _err; code = tsdbAppendLastBlock(pCommitter);
TSDB_CHECK_CODE(code, lino, _exit);
}
} }
} }
_exit: _exit:
return code; if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
_err: tstrerror(code));
tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); }
return code; return code;
} }
static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SRowInfo *pRowInfo; SRowInfo *pRowInfo;
TABLEID id = {0}; TABLEID id = {0};
...@@ -1421,36 +1583,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { ...@@ -1421,36 +1583,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
id.uid = pRowInfo->uid; id.uid = pRowInfo->uid;
code = tsdbMoveCommitData(pCommitter, id); code = tsdbMoveCommitData(pCommitter, id);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// start // start
tMapDataReset(&pCommitter->dWriter.mBlock); tMapDataReset(&pCommitter->dWriter.mBlock);
// impl // impl
code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
/* merge with data in .data file */ /* merge with data in .data file */
code = tsdbMergeTableData(pCommitter, id); code = tsdbMergeTableData(pCommitter, id);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
/* handle remain table data */ /* handle remain table data */
code = tsdbCommitTableData(pCommitter, id); code = tsdbCommitTableData(pCommitter, id);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
// end // end
if (pCommitter->dWriter.mBlock.nItem > 0) { if (pCommitter->dWriter.mBlock.nItem > 0) {
SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid};
code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _err; TSDB_CHECK_CODE(code, lino, _exit);
} }
} }
} }
...@@ -1458,15 +1620,20 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { ...@@ -1458,15 +1620,20 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
id.suid = INT64_MAX; id.suid = INT64_MAX;
id.uid = INT64_MAX; id.uid = INT64_MAX;
code = tsdbMoveCommitData(pCommitter, id); code = tsdbMoveCommitData(pCommitter, id);
if (code) goto _err; TSDB_CHECK_CODE(code, lino, _exit);
#if USE_STREAM_COMPRESSION
code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
#else
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
pCommitter->cmprAlg); pCommitter->cmprAlg);
if (code) goto _err; #endif
TSDB_CHECK_CODE(code, lino, _exit);
return code;
_err: _exit:
tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); if (code) {
tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
tstrerror(code));
}
return code; return code;
} }
source/dnode/vnode/src/tsdb/tsdbDiskData.c
浏览文件 @ 05a39977
...@@ -16,69 +16,682 @@ ...@@ -16,69 +16,682 @@
#include "tsdb.h" #include "tsdb.h"
typedef struct SDiskColBuilder SDiskColBuilder; typedef struct SDiskColBuilder SDiskColBuilder;
struct SDiskColBuilder { struct SDiskColBuilder {
uint8_t flags; int16_t cid;
uint8_t *pBitMap; int8_t type;
int32_t *aOffset; uint8_t cmprAlg;
int32_t nData; uint8_t calcSma;
uint8_t *pData; int8_t flag;
int32_t nVal;
uint8_t *pBitMap;
int32_t offset;
SCompressor *pOffC;
SCompressor *pValC;
SColumnDataAgg sma;
uint8_t minSet;
uint8_t maxSet;
uint8_t *aBuf[2];
}; };
int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { // SDiskData ================================================
static int32_t tDiskDataDestroy(SDiskData *pDiskData) {
int32_t code = 0;
pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol);
return code;
}
// SDiskColBuilder ================================================
#define tDiskColBuilderCreate() \
(SDiskColBuilder) { 0 }
static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) {
int32_t code = 0;
tFree(pBuilder->pBitMap);
if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC);
if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC);
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
tFree(pBuilder->aBuf[iBuf]);
}
return code;
}
static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg,
uint8_t calcSma) {
int32_t code = 0;
pBuilder->cid = cid;
pBuilder->type = type;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma;
pBuilder->flag = 0;
pBuilder->nVal = 0;
pBuilder->offset = 0;
if (IS_VAR_DATA_TYPE(type)) {
if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code;
code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg);
if (code) return code;
}
if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code;
code = tCompressStart(pBuilder->pValC, type, cmprAlg);
if (code) return code;
if (pBuilder->calcSma) {
pBuilder->sma = (SColumnDataAgg){.colId = cid};
pBuilder->minSet = 0;
pBuilder->maxSet = 0;
}
return code;
}
static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
int32_t code = 0;
ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE);
*pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid,
.type = pBuilder->type,
.smaOn = pBuilder->calcSma,
.flag = pBuilder->flag,
.szOrigin = 0,
.szBitmap = 0,
.szOffset = 0,
.szValue = 0,
.offset = 0},
.pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma};
if (pBuilder->flag == HAS_NULL) return code;
// BITMAP
if (pBuilder->flag != HAS_VALUE) {
int32_t nBit;
if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
nBit = BIT2_SIZE(pBuilder->nVal);
} else {
nBit = BIT1_SIZE(pBuilder->nVal);
}
code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
pDiskCol->bCol.szBitmap =
tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES,
pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
pDiskCol->pBit = pBuilder->aBuf[0];
}
// OFFSET
if (IS_VAR_DATA_TYPE(pBuilder->type)) {
code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL);
if (code) return code;
}
// VALUE
if (pBuilder->flag != (HAS_NULL | HAS_NONE)) {
code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin);
if (code) return code;
}
return code;
}
static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0; int32_t code = 0;
// TODO
if (IS_VAR_DATA_TYPE(pColVal->type)) {
code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t));
if (code) return code;
pBuilder->offset += pColVal->value.nData;
code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData);
if (code) return code;
} else {
code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes);
if (code) return code;
}
return code; return code;
} }
static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_VALUE;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NONE;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NULL;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
// bit map
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
// value
pBuilder->flag |= HAS_VALUE;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
pBuilder->flag |= HAS_NULL;
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_VALUE;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_NONE;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_VALUE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal));
}
SET_BIT2(pBitMap, pBuilder->nVal, 2);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) {
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0);
}
SET_BIT2(pBitMap, pBuilder->nVal, 1);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pBuilder->nVal, 0);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
// ================================================================ code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
typedef struct SDiskDataBuilder SDiskDataBuilder; if (code) return code;
struct SDiskDataBuilder { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0);
SDiskDataHdr hdr;
SArray *aBlockCol; // SArray<SBlockCol> return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = {
{tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0
{tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE
{tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL
{tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE
{tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE
{tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE
{tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL
{tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE
}; };
extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (pBuilder->calcSma) {
if (COL_VAL_IS_VALUE(pColVal)) {
tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
} else {
pBuilder->sma.numOfNull++;
}
}
if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) {
code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal);
if (code) return code;
}
pBuilder->nVal++;
return code;
}
// SDiskDataBuilder ================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
int32_t code = 0; int32_t code = 0;
// TODO
*ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder));
if (*ppBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
return code; return code;
} }
void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
// TODO if (pBuilder == NULL) return NULL;
if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC);
if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC);
if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC);
if (pBuilder->aBuilder) {
for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
tDiskColBuilderDestroy(pDCBuilder);
}
taosArrayDestroy(pBuilder->aBuilder);
}
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
tFree(pBuilder->aBuf[iBuf]);
}
tDiskDataDestroy(&pBuilder->dd);
taosMemoryFree(pBuilder);
return NULL;
} }
void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // uint8_t calcSma) {
.fmtVer = 0, int32_t code = 0;
.suid = suid,
.uid = uid, ASSERT(pId->suid || pId->uid);
.cmprAlg = cmprAlg};
pBuilder->suid = pId->suid;
pBuilder->uid = pId->uid;
pBuilder->nRow = 0;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = calcSma;
pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX,
.maxUid = INT64_MIN,
.minKey = TSKEY_MAX,
.maxKey = TSKEY_MIN,
.minVer = VERSION_MAX,
.maxVer = VERSION_MIN,
.minTKey = TSDBKEY_MAX,
.maxTKey = TSDBKEY_MIN};
if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code;
code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code;
code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code;
code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg);
if (code) return code;
if (pBuilder->aBuilder == NULL) {
pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder));
if (pBuilder->aBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
pBuilder->nBuilder = 0;
for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
STColumn *pTColumn = &pTSchema->columns[iCol];
if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) {
SDiskColBuilder dc = tDiskColBuilderCreate();
if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder);
code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg,
(calcSma && (pTColumn->flags & COL_SMA_ON)));
if (code) return code;
pBuilder->nBuilder++;
}
return code;
} }
void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) {
// TODO int32_t code = 0;
pBuilder->suid = 0;
pBuilder->uid = 0;
pBuilder->nRow = 0;
return code;
} }
int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) {
int32_t code = 0; int32_t code = 0;
// uid (todo) ASSERT(pBuilder->suid || pBuilder->uid);
ASSERT(pId->suid == pBuilder->suid);
TSDBKEY kRow = TSDBROW_KEY(pRow);
if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow;
if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow;
// version (todo) // uid
if (pBuilder->uid && pBuilder->uid != pId->uid) {
ASSERT(pBuilder->suid);
for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) {
code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t));
if (code) return code;
}
pBuilder->uid = 0;
}
if (pBuilder->uid == 0) {
code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t));
if (code) return code;
}
if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid;
if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid;
// TSKEY (todo) // version
code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version;
if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version;
// TSKEY
code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts;
if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts;
SRowIter iter = {0}; SRowIter iter = {0};
tRowIterInit(&iter, pRow, pTSchema); tRowIterInit(&iter, pRow, pTSchema);
for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { SColVal *pColVal = tRowIterNext(&iter);
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
while (pColVal && pColVal->cid < pDCBuilder->cid) {
pColVal = tRowIterNext(&iter);
}
if (pColVal && pColVal->cid == pDCBuilder->cid) {
code = tDiskColAddVal(pDCBuilder, pColVal);
if (code) return code;
pColVal = tRowIterNext(&iter);
} else {
code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type));
if (code) return code;
}
} }
pBuilder->nRow++;
return code; return code;
} }
int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) {
int32_t code = 0; int32_t code = 0;
// TODO
ASSERT(pBuilder->nRow);
*ppDiskData = NULL;
*ppBlkInfo = NULL;
SDiskData *pDiskData = &pBuilder->dd;
// reset SDiskData
pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT,
.fmtVer = 0,
.suid = pBuilder->suid,
.uid = pBuilder->uid,
.szUid = 0,
.szVer = 0,
.szKey = 0,
.szBlkCol = 0,
.nRow = pBuilder->nRow,
.cmprAlg = pBuilder->cmprAlg};
pDiskData->pUid = NULL;
pDiskData->pVer = NULL;
pDiskData->pKey = NULL;
// UID
if (pBuilder->uid == 0) {
code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL);
if (code) return code;
}
// VERSION
code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL);
if (code) return code;
// TSKEY
code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL);
if (code) return code;
// aDiskCol
if (pDiskData->aDiskCol) {
taosArrayClear(pDiskData->aDiskCol);
} else {
pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol));
if (pDiskData->aDiskCol == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
int32_t offset = 0;
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
if (pDCBuilder->flag == HAS_NONE) continue;
SDiskCol dCol;
code = tGnrtDiskCol(pDCBuilder, &dCol);
if (code) return code;
dCol.bCol.offset = offset;
offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue;
if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol);
}
*ppDiskData = pDiskData;
*ppBlkInfo = &pBuilder->bi;
return code; return code;
} }
\ No newline at end of file
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @ 05a39977
...@@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn ...@@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
} }
} }
// fill the mis-matched columns with null value
while (i < numOfOutputCols) { while (i < numOfOutputCols) {
pColData = taosArrayGet(pResBlock->pDataBlock, i); pColData = taosArrayGet(pResBlock->pDataBlock, i);
colDataAppendNNULL(pColData, 0, remain); colDataAppendNNULL(pColData, 0, remain);
...@@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn ...@@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
pResBlock->info.rows = remain; pResBlock->info.rows = remain;
pDumpInfo->rowIndex += step * remain; pDumpInfo->rowIndex += step * remain;
// check if current block are all handled
if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) {
// int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
// setBlockAllDumped(pDumpInfo, ts, pReader->order); if (outOfTimeWindow(ts, &pReader->window)) { // the remain data has out of query time window, ignore current block
setBlockAllDumped(pDumpInfo, ts, pReader->order);
}
} else { } else {
int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; int64_t ts = asc ? pBlock->maxKey.ts : pBlock->minKey.ts;
setBlockAllDumped(pDumpInfo, k, pReader->order); setBlockAllDumped(pDumpInfo, ts, pReader->order);
} }
double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
......
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
浏览文件 @ 05a39977
...@@ -128,7 +128,7 @@ _exit: ...@@ -128,7 +128,7 @@ _exit:
return code; return code;
} }
static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) {
int32_t code = 0; int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
...@@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, ...@@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
// write // write
if (pSmaInfo->size) { if (pSmaInfo->size) {
code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size);
if (code) goto _err;
code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size);
if (code) goto _err; if (code) goto _err;
...@@ -607,6 +604,132 @@ _err: ...@@ -607,6 +604,132 @@ _err:
return code; return code;
} }
int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) {
int32_t code = 0;
int32_t lino = 0;
STsdbFD *pFD = NULL;
if (pSmaInfo) {
pFD = pWriter->pDataFD;
pBlkInfo->offset = pWriter->fData.size;
} else {
pFD = pWriter->pSttFD;
pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size;
}
pBlkInfo->szBlock = 0;
pBlkInfo->szKey = 0;
// hdr
int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr);
code = tRealloc(&pWriter->aBuf[0], n);
TSDB_CHECK_CODE(code, lino, _exit);
tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr);
code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += n;
pBlkInfo->szBlock += n;
// uid + ver + key
if (pDiskData->pUid) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szUid;
pBlkInfo->szBlock += pDiskData->hdr.szUid;
}
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szVer;
pBlkInfo->szBlock += pDiskData->hdr.szVer;
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szKey += pDiskData->hdr.szKey;
pBlkInfo->szBlock += pDiskData->hdr.szKey;
// aBlockCol
if (pDiskData->hdr.szBlkCol) {
code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
n = 0;
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol);
}
ASSERT(n == pDiskData->hdr.szBlkCol);
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskData->hdr.szBlkCol;
}
// aDiskCol
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
if (pDiskCol->pBit) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szBitmap;
}
if (pDiskCol->pOff) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szOffset;
}
if (pDiskCol->pVal) {
code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue);
TSDB_CHECK_CODE(code, lino, _exit);
pBlkInfo->szBlock += pDiskCol->bCol.szValue;
}
}
if (pSmaInfo) {
pWriter->fData.size += pBlkInfo->szBlock;
} else {
pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock;
goto _exit;
}
pSmaInfo->offset = 0;
pSmaInfo->size = 0;
for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue;
if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue;
if (!pDiskCol->bCol.smaOn) continue;
code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg));
TSDB_CHECK_CODE(code, lino, _exit);
pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg);
}
if (pSmaInfo->size) {
pSmaInfo->offset = pWriter->fSma.size;
code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size);
TSDB_CHECK_CODE(code, lino, _exit);
pWriter->fSma.size += pSmaInfo->size;
}
_exit:
if (code) {
tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
}
return code;
}
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
int32_t code = 0; int32_t code = 0;
int64_t n; int64_t n;
...@@ -946,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ...@@ -946,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
ASSERT(pBlockData->suid == hdr.suid); ASSERT(pBlockData->suid == hdr.suid);
ASSERT(pBlockData->uid == hdr.uid);
pBlockData->uid = hdr.uid;
pBlockData->nRow = hdr.nRow; pBlockData->nRow = hdr.nRow;
// uid // uid
......
source/dnode/vnode/src/tsdb/tsdbUtil.c
浏览文件 @ 05a39977
...@@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo ...@@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
...@@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { ...@@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
STColumn *pTColumn; STColumn *pTColumn;
int32_t iCol, jCol = 1; int32_t iCol, jCol = 1;
ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol]; pTColumn = &pMerger->pTSchema->columns[iCol];
...@@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { ...@@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
*pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
...@@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { ...@@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
TSDBKEY key = TSDBROW_KEY(pRow); TSDBKEY key = TSDBROW_KEY(pRow);
SColVal *pColVal = &(SColVal){0}; SColVal *pColVal = &(SColVal){0};
ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal);
...@@ -1105,9 +1105,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS ...@@ -1105,9 +1105,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
pColVal = tRowIterNext(&rIter); pColVal = tRowIterNext(&rIter);
} }
} }
_exit:
pBlockData->nRow++; pBlockData->nRow++;
return code; return code;
_err: _err:
...@@ -1455,9 +1454,8 @@ _exit: ...@@ -1455,9 +1454,8 @@ _exit:
} }
// SDiskDataHdr ============================== // SDiskDataHdr ==============================
int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) {
int32_t n = 0; int32_t n = 0;
SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;
n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32(p ? p + n : p, pHdr->delimiter);
n += tPutU32v(p ? p + n : p, pHdr->fmtVer); n += tPutU32v(p ? p + n : p, pHdr->fmtVer);
...@@ -1516,174 +1514,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { ...@@ -1516,174 +1514,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
return n; return n;
} }
#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \
do { \
(SUM_V) += (VAL); \
if (!(MINSET)) { \
(MIN_V) = (VAL); \
(MINSET) = 1; \
} else if ((MIN_V) > (VAL)) { \
(MIN_V) = (VAL); \
} \
if (!(MAXSET)) { \
(MAX_V) = (VAL); \
(MAXSET) = 1; \
} else if ((MAX_V) < (VAL)) { \
(MAX_V) = (VAL); \
} \
} while (0)
static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int8_t val = *(int8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
int16_t val = *(int16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int32_t val = *(int32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
int64_t val = *(int64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
float val = *(float *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
double val = *(double *)&pColVal->value.val;
SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint8_t val = *(uint8_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint16_t val = *(uint16_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
uint32_t val = *(uint32_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
uint8_t *maxSet) {
uint64_t val = *(uint64_t *)&pColVal->value.val;
SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
}
void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = {
NULL,
tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL
tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT
tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT
tSmaUpdateInt, // TSDB_DATA_TYPE_INT
tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT
tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT
tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE
NULL, // TSDB_DATA_TYPE_VARCHAR
tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP
NULL, // TSDB_DATA_TYPE_NCHAR
tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT
tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT
tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT
tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT
NULL, // TSDB_DATA_TYPE_JSON
NULL, // TSDB_DATA_TYPE_VARBINARY
NULL, // TSDB_DATA_TYPE_DECIMAL
NULL, // TSDB_DATA_TYPE_BLOB
NULL, // TSDB_DATA_TYPE_MEDIUMBLOB
};
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
SColVal colVal;
SColVal *pColVal = &colVal;
memset(pColAgg, 0, sizeof(*pColAgg));
bool minAssigned = false;
bool maxAssigned = false;
*pColAgg = (SColumnDataAgg){.colId = pColData->cid}; *pColAgg = (SColumnDataAgg){.colId = pColData->cid};
uint8_t minSet = 0;
uint8_t maxSet = 0;
SColVal cv;
for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
tColDataGetValue(pColData, iVal, pColVal); tColDataGetValue(pColData, iVal, &cv);
if (!COL_VAL_IS_VALUE(pColVal)) { if (COL_VAL_IS_VALUE(&cv)) {
pColAgg->numOfNull++; tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet);
} else { } else {
switch (pColData->type) { pColAgg->numOfNull++;
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
break;
case TSDB_DATA_TYPE_TINYINT: {
pColAgg->sum += colVal.value.i8;
if (!minAssigned || pColAgg->min > colVal.value.i8) {
pColAgg->min = colVal.value.i8;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i8) {
pColAgg->max = colVal.value.i8;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_SMALLINT: {
pColAgg->sum += colVal.value.i16;
if (!minAssigned || pColAgg->min > colVal.value.i16) {
pColAgg->min = colVal.value.i16;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i16) {
pColAgg->max = colVal.value.i16;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_INT: {
pColAgg->sum += colVal.value.i32;
if (!minAssigned || pColAgg->min > colVal.value.i32) {
pColAgg->min = colVal.value.i32;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i32) {
pColAgg->max = colVal.value.i32;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_BIGINT: {
pColAgg->sum += colVal.value.i64;
if (!minAssigned || pColAgg->min > colVal.value.i64) {
pColAgg->min = colVal.value.i64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i64) {
pColAgg->max = colVal.value.i64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_FLOAT: {
*(double *)(&pColAgg->sum) += colVal.value.f;
if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) {
*(double *)(&pColAgg->min) = colVal.value.f;
minAssigned = true;
}
if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) {
*(double *)(&pColAgg->max) = colVal.value.f;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_DOUBLE: {
*(double *)(&pColAgg->sum) += colVal.value.d;
if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) {
*(double *)(&pColAgg->min) = colVal.value.d;
minAssigned = true;
}
if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) {
*(double *)(&pColAgg->max) = colVal.value.d;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_VARCHAR:
break;
case TSDB_DATA_TYPE_TIMESTAMP: {
if (!minAssigned || pColAgg->min > colVal.value.i64) {
pColAgg->min = colVal.value.i64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.i64) {
pColAgg->max = colVal.value.i64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_NCHAR:
break;
case TSDB_DATA_TYPE_UTINYINT: {
pColAgg->sum += colVal.value.u8;
if (!minAssigned || pColAgg->min > colVal.value.u8) {
pColAgg->min = colVal.value.u8;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u8) {
pColAgg->max = colVal.value.u8;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_USMALLINT: {
pColAgg->sum += colVal.value.u16;
if (!minAssigned || pColAgg->min > colVal.value.u16) {
pColAgg->min = colVal.value.u16;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u16) {
pColAgg->max = colVal.value.u16;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_UINT: {
pColAgg->sum += colVal.value.u32;
if (!minAssigned || pColAgg->min > colVal.value.u32) {
pColAgg->min = colVal.value.u32;
minAssigned = true;
}
if (!minAssigned || pColAgg->max < colVal.value.u32) {
pColAgg->max = colVal.value.u32;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_UBIGINT: {
pColAgg->sum += colVal.value.u64;
if (!minAssigned || pColAgg->min > colVal.value.u64) {
pColAgg->min = colVal.value.u64;
minAssigned = true;
}
if (!maxAssigned || pColAgg->max < colVal.value.u64) {
pColAgg->max = colVal.value.u64;
maxAssigned = true;
}
break;
}
case TSDB_DATA_TYPE_JSON:
break;
case TSDB_DATA_TYPE_VARBINARY:
break;
case TSDB_DATA_TYPE_DECIMAL:
break;
case TSDB_DATA_TYPE_BLOB:
break;
case TSDB_DATA_TYPE_MEDIUMBLOB:
break;
default:
ASSERT(0);
}
} }
} }
} }
......
source/dnode/vnode/src/vnd/vnodeSync.c
浏览文件 @ 05a39977
...@@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
const STraceId *trace = &pMsg->info.traceId; const STraceId *trace = &pMsg->info.traceId;
if (!syncEnvIsStart()) { if (!syncEnvIsStart()) {
vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId); vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg);
terrno = TSDB_CODE_APP_ERROR; terrno = TSDB_CODE_APP_ERROR;
return -1; return -1;
} }
SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
if (pSyncNode == NULL) { if (pSyncNode == NULL) {
vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId); vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR; terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1; return -1;
} }
...@@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SRpcMsg rsp = {.code = code, .info = pMsg->info}; SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} else { } else {
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType);
code = -1; code = -1;
} }
...@@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { ...@@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
SRpcMsg rsp = {.code = code, .info = pMsg->info}; SRpcMsg rsp = {.code = code, .info = pMsg->info};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} else { } else {
vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType);
code = -1; code = -1;
} }
} }
...@@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void ...@@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void
vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId); vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId);
break; break;
} else { } else {
vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId); vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId, itemSize);
taosMsleep(10); taosMsleep(10);
} }
} while (true); } while (true);
...@@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { ...@@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) {
vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId);
break; break;
} else { } else {
vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize);
taosMsleep(10); taosMsleep(10);
} }
} while (true); } while (true);
......
source/libs/executor/src/executil.c
浏览文件 @ 05a39977
...@@ -420,7 +420,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray ...@@ -420,7 +420,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray
goto end; goto end;
} }
} else { } else {
metaGetTableTagsOpt(metaHandle, suid, uidList, tags); metaGetTableTagsByUids(metaHandle, suid, uidList, tags);
qInfo("succ to get table from meta idx, suid:%" PRIu64, suid); qInfo("succ to get table from meta idx, suid:%" PRIu64, suid);
} }
......
source/libs/executor/src/scanoperator.c
浏览文件 @ 05a39977
...@@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S ...@@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S
static bool processBlockWithProbability(const SSampleExecInfo* pInfo); static bool processBlockWithProbability(const SSampleExecInfo* pInfo);
static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable,
const char* tableName, int32_t* pNumOfRows, SMetaReader* smrChildTable, const char* dbname, const char* tableName,
const SSDataBlock* dataBlock); int32_t* pNumOfRows, const SSDataBlock* dataBlock);
static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock); static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock);
bool processBlockWithProbability(const SSampleExecInfo* pInfo) { bool processBlockWithProbability(const SSampleExecInfo* pInfo) {
...@@ -2415,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { ...@@ -2415,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) {
strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 &&
nodeType(node->pRight) == QUERY_NODE_VALUE) { nodeType(node->pRight) == QUERY_NODE_VALUE) {
SValueNode* pValue = (SValueNode*)node->pRight; SValueNode* pValue = (SValueNode*)node->pRight;
if (pValue->node.type == TSDB_DATA_TYPE_NCHAR || pValue->node.type == TSDB_DATA_TYPE_VARCHAR || if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR ||
pValue->node.type == TSDB_DATA_TYPE_BINARY) { pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) {
char* value = nodesGetStrValueFromNode(pValue); char* value = nodesGetValueFromNode(pValue);
strncpy(condTable, value, TSDB_TABLE_NAME_LEN); strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN);
taosMemoryFree(value);
return true; return true;
} }
} }
...@@ -2480,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { ...@@ -2480,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tableName, condTableName); STR_TO_VARSTR(tableName, condTableName);
SMetaReader smr = {0}; SMetaReader smrChildTable = {0};
metaReaderInit(&smr, pInfo->readHandle.meta, 0); metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0);
metaGetTableEntryByName(&smr, condTableName); metaGetTableEntryByName(&smrChildTable, condTableName);
sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); if (smrChildTable.me.type != TSDB_CHILD_TABLE) {
metaReaderClear(&smr); metaReaderClear(&smrChildTable);
blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows = 0;
return NULL;
}
SMetaReader smrSuperTable = {0};
metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid);
sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smrSuperTable);
metaReaderClear(&smrChildTable);
if (numOfRows > 0) { if (numOfRows > 0) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock);
numOfRows = 0; numOfRows = 0;
} }
blockDataDestroy(dataBlock); blockDataDestroy(dataBlock);
pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; pInfo->loadInfo.totalRows += pInfo->pRes->info.rows;
doSetOperatorCompleted(pOperator);
return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes;
} }
...@@ -2508,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { ...@@ -2508,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) {
char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name);
SMetaReader smr = {0}; SMetaReader smrSuperTable = {0};
metaReaderInit(&smr, pInfo->readHandle.meta, 0); metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0);
uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid;
int32_t code = metaGetTableEntryByUid(&smr, suid); int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno),
GET_TASKID(pTaskInfo)); GET_TASKID(pTaskInfo));
metaReaderClear(&smr); metaReaderClear(&smrSuperTable);
metaCloseTbCursor(pInfo->pCur); metaCloseTbCursor(pInfo->pCur);
pInfo->pCur = NULL; pInfo->pCur = NULL;
T_LONG_JMP(pTaskInfo->env, terrno); T_LONG_JMP(pTaskInfo->env, terrno);
} }
sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock);
metaReaderClear(&smr); metaReaderClear(&smrSuperTable);
if (numOfRows >= pOperator->resultInfo.capacity) { if (numOfRows >= pOperator->resultInfo.capacity) {
relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock);
...@@ -2562,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t ...@@ -2562,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t
blockDataCleanup(dataBlock); blockDataCleanup(dataBlock);
} }
static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable,
const char* tableName, int32_t* pNumOfRows, SMetaReader* smrChildTable, const char* dbname, const char* tableName,
const SSDataBlock* dataBlock) { int32_t* pNumOfRows, const SSDataBlock* dataBlock) {
char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(stableName, (*smr).me.name); STR_TO_VARSTR(stableName, (*smrSuperTable).me.name);
int32_t numOfRows = *pNumOfRows; int32_t numOfRows = *pNumOfRows;
int32_t numOfTags = (*smr).me.stbEntry.schemaTag.nCols; int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols;
for (int32_t i = 0; i < numOfTags; ++i) { for (int32_t i = 0; i < numOfTags; ++i) {
SColumnInfoData* pColInfoData = NULL; SColumnInfoData* pColInfoData = NULL;
...@@ -2588,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, ...@@ -2588,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo,
// tag name // tag name
char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(tagName, (*smr).me.stbEntry.schemaTag.pSchema[i].name); STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name);
pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3);
colDataAppend(pColInfoData, numOfRows, tagName, false); colDataAppend(pColInfoData, numOfRows, tagName, false);
// tag type // tag type
int8_t tagType = (*smr).me.stbEntry.schemaTag.pSchema[i].type; int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type;
pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4);
char tagTypeStr[VARSTR_HEADER_SIZE + 32]; char tagTypeStr[VARSTR_HEADER_SIZE + 32];
int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name);
if (tagType == TSDB_DATA_TYPE_VARCHAR) { if (tagType == TSDB_DATA_TYPE_VARCHAR) {
tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
(int32_t)((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE));
} else if (tagType == TSDB_DATA_TYPE_NCHAR) { } else if (tagType == TSDB_DATA_TYPE_NCHAR) {
tagTypeLen += tagTypeLen += sprintf(
sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", varDataVal(tagTypeStr) + tagTypeLen, "(%d)",
(int32_t)(((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE));
} }
varDataSetLen(tagTypeStr, tagTypeLen); varDataSetLen(tagTypeStr, tagTypeLen);
colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false);
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = (*smr).me.stbEntry.schemaTag.pSchema[i].colId; tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId;
char* tagData = NULL; char* tagData = NULL;
uint32_t tagLen = 0; uint32_t tagLen = 0;
if (tagType == TSDB_DATA_TYPE_JSON) { if (tagType == TSDB_DATA_TYPE_JSON) {
tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; tagData = (char*)smrChildTable->me.ctbEntry.pTags;
} else { } else {
bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal);
if (exist) { if (exist) {
if (IS_VAR_DATA_TYPE(tagType)) { if (IS_VAR_DATA_TYPE(tagType)) {
tagData = (char*)tagVal.pData; tagData = (char*)tagVal.pData;
......
source/libs/executor/src/timewindowoperator.c
浏览文件 @ 05a39977
...@@ -1402,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 ...@@ -1402,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3
SResultRowInfo dumyInfo; SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1; dumyInfo.cur.pageId = -1;
STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC);
while (win.skey <= endTsCols[i]) { do {
uint64_t winGpId = pGpDatas[i]; uint64_t winGpId = pGpDatas[i];
bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput);
SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; SWinKey winRes = {.ts = win.skey, .groupId = winGpId};
...@@ -1413,7 +1413,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 ...@@ -1413,7 +1413,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3
taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey));
} }
getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win);
} } while (win.ekey <= endTsCols[i]);
} }
} }
...@@ -3067,8 +3067,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p ...@@ -3067,8 +3067,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
int32_t startPos = 0; int32_t startPos = 0;
TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols);
STimeWindow nextWin = STimeWindow nextWin = {0};
getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); if (IS_FINAL_OP(pInfo)) {
nextWin = getFinalTimeWindow(ts, &pInfo->interval);
} else {
nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
}
while (1) { while (1) {
bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) {
...@@ -3122,8 +3126,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p ...@@ -3122,8 +3126,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, if (IS_FINAL_OP(pInfo)) {
TSDB_ORDER_ASC); forwardRows = 1;
} else {
forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey,
NULL, TSDB_ORDER_ASC);
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) {
saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap);
} }
......
source/libs/index/src/indexFilter.c
浏览文件 @ 05a39977
...@@ -468,6 +468,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP ...@@ -468,6 +468,7 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP
SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST); SIndexMultiTermQuery *mtm = indexMultiTermQueryCreate(MUST);
indexMultiTermQueryAdd(mtm, tm, qtype); indexMultiTermQueryAdd(mtm, tm, qtype);
ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); ret = indexJsonSearch(arg->ivtIdx, mtm, output->result);
indexMultiTermQueryDestroy(mtm);
} else { } else {
bool reverse; bool reverse;
FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse);
...@@ -647,9 +648,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { ...@@ -647,9 +648,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) {
SIF_ERR_RET(sifInitOperParams(&params, node, ctx)); SIF_ERR_RET(sifInitOperParams(&params, node, ctx));
if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { if (params[0].status == SFLT_NOT_INDEX && (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) {
for (int i = 0; i < nParam; i++) sifFreeParam(&params[i]);
output->status = SFLT_NOT_INDEX; output->status = SFLT_NOT_INDEX;
return code; goto _return;
} }
// ugly code, refactor later // ugly code, refactor later
......
source/libs/index/src/indexFst.c
浏览文件 @ 05a39977
...@@ -1025,6 +1025,7 @@ void fstDestroy(Fst* fst) { ...@@ -1025,6 +1025,7 @@ void fstDestroy(Fst* fst) {
} }
bool fstGet(Fst* fst, FstSlice* b, Output* out) { bool fstGet(Fst* fst, FstSlice* b, Output* out) {
int ret = false;
FstNode* root = fstGetRoot(fst); FstNode* root = fstGetRoot(fst);
Output tOut = 0; Output tOut = 0;
int32_t len; int32_t len;
...@@ -1037,7 +1038,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { ...@@ -1037,7 +1038,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
uint8_t inp = data[i]; uint8_t inp = data[i];
Output res = 0; Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) { if (false == fstNodeFindInput(root, inp, &res)) {
return false; goto _return;
} }
FstTransition trn; FstTransition trn;
...@@ -1047,18 +1048,20 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) { ...@@ -1047,18 +1048,20 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
taosArrayPush(nodes, &root); taosArrayPush(nodes, &root);
} }
if (!FST_NODE_IS_FINAL(root)) { if (!FST_NODE_IS_FINAL(root)) {
return false; goto _return;
} else { } else {
tOut = tOut + FST_NODE_FINAL_OUTPUT(root); tOut = tOut + FST_NODE_FINAL_OUTPUT(root);
ret = true;
} }
_return:
for (int32_t i = 0; i < taosArrayGetSize(nodes); i++) { for (int32_t i = 0; i < taosArrayGetSize(nodes); i++) {
FstNode** node = (FstNode**)taosArrayGet(nodes, i); FstNode** node = (FstNode**)taosArrayGet(nodes, i);
fstNodeDestroy(*node); fstNodeDestroy(*node);
} }
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
*out = tOut; *out = tOut;
return true; return ret;
} }
FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx) { FStmBuilder* fstSearch(Fst* fst, FAutoCtx* ctx) {
// refactor later // refactor later
...@@ -1243,6 +1246,7 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) { ...@@ -1243,6 +1246,7 @@ bool stmStSeekMin(FStmSt* sws, FstBoundWithData* min) {
StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
taosArrayPush(sws->stack, &s); taosArrayPush(sws->stack, &s);
taosMemoryFree(trans);
return true; return true;
} }
} }
......
source/libs/index/src/indexFstDfa.c
浏览文件 @ 05a39977
...@@ -139,6 +139,7 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r ...@@ -139,6 +139,7 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r
} }
} }
if (taosArrayGetSize(tinsts) == 0) { if (taosArrayGetSize(tinsts) == 0) {
taosArrayDestroy(tinsts);
return false; return false;
} }
uint32_t *v = taosHashGet(builder->cache, &tinsts, sizeof(POINTER_BYTES)); uint32_t *v = taosHashGet(builder->cache, &tinsts, sizeof(POINTER_BYTES));
......
source/libs/index/src/indexTfile.c
浏览文件 @ 05a39977
...@@ -267,10 +267,12 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { ...@@ -267,10 +267,12 @@ static int32_t tfSearchPrefix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i); uint64_t offset = *(uint64_t*)taosArrayGet(offsets, i);
ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total); ret = tfileReaderLoadTableIds((TFileReader*)reader, offset, tr->total);
if (ret != 0) { if (ret != 0) {
taosArrayDestroy(offsets);
indexError("failed to find target tablelist"); indexError("failed to find target tablelist");
return TSDB_CODE_TDB_FILE_CORRUPTED; return TSDB_CODE_TDB_FILE_CORRUPTED;
} }
} }
taosArrayDestroy(offsets);
return 0; return 0;
} }
static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { static int32_t tfSearchSuffix(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
...@@ -336,6 +338,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, ...@@ -336,6 +338,7 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr,
} }
stmStDestroy(st); stmStDestroy(st);
stmBuilderDestroy(sb); stmBuilderDestroy(sb);
taosArrayDestroy(offsets);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { static int32_t tfSearchLessThan(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
...@@ -379,6 +382,7 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { ...@@ -379,6 +382,7 @@ static int32_t tfSearchTerm_JSON(void* reader, SIndexTerm* tem, SIdxTRslt* tr) {
", size: %d, time cost: %" PRIu64 "us", ", size: %d, time cost: %" PRIu64 "us",
tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost); tem->suid, tem->colName, tem->colVal, offset, (int)taosArrayGetSize(tr->total), cost);
} }
taosMemoryFree(p);
fstSliceDestroy(&key); fstSliceDestroy(&key);
return 0; return 0;
} }
...@@ -471,6 +475,9 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt ...@@ -471,6 +475,9 @@ static int32_t tfSearchCompareFunc_JSON(void* reader, SIndexTerm* tem, SIdxTRslt
} }
stmStDestroy(st); stmStDestroy(st);
stmBuilderDestroy(sb); stmBuilderDestroy(sb);
taosArrayDestroy(offsets);
taosMemoryFree(p);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) { int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr) {
...@@ -898,9 +905,8 @@ static int tfileReaderLoadFst(TFileReader* reader) { ...@@ -898,9 +905,8 @@ static int tfileReaderLoadFst(TFileReader* reader) {
int64_t ts = taosGetTimestampUs(); int64_t ts = taosGetTimestampUs();
int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset); int32_t nread = ctx->readFrom(ctx, buf, fstSize, reader->header.fstOffset);
int64_t cost = taosGetTimestampUs() - ts; int64_t cost = taosGetTimestampUs() - ts;
indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %" PRId64 ", time cost: %" PRId64 indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread,
"us", reader->header.fstOffset, fstSize, ctx->file.buf, size, cost);
nread, reader->header.fstOffset, fstSize, ctx->file.buf, size, cost);
// we assuse fst size less than FST_MAX_SIZE // we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread <= fstSize); assert(nread > 0 && nread <= fstSize);
...@@ -989,6 +995,7 @@ static SArray* tfileGetFileList(const char* path) { ...@@ -989,6 +995,7 @@ static SArray* tfileGetFileList(const char* path) {
TdDirPtr pDir = taosOpenDir(path); TdDirPtr pDir = taosOpenDir(path);
if (NULL == pDir) { if (NULL == pDir) {
taosArrayDestroy(files);
return NULL; return NULL;
} }
TdDirEntryPtr pDirEntry; TdDirEntryPtr pDirEntry;
......
source/libs/parser/inc/parUtil.h
浏览文件 @ 05a39977
...@@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); ...@@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta);
STableMeta* tableMetaDup(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta);
int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName);
int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq);
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache,
......
source/libs/parser/src/parAstParser.c
浏览文件 @ 05a39977
...@@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) { ...@@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) {
return false; return false;
} }
static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pName) {
if (TSDB_DB_NAME_T == pName->type) {
return reserveDbVgInfoInCache(pName->acctId, pName->dbname, pCxt->pMetaCache);
}
return reserveTableVgroupInCacheExt(pName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) {
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt;
SName name = {0};
int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name);
if (TSDB_CODE_SUCCESS == code) {
code = collectMetaKeyFromInsTagsImpl(pCxt, &name);
}
return code;
}
static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable,
AUTH_TYPE authType) { AUTH_TYPE authType) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache);
...@@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c ...@@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) {
code = reserveDnodeRequiredInCache(pCxt->pMetaCache); code = reserveDnodeRequiredInCache(pCxt->pMetaCache);
} }
if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) &&
QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) {
code = collectMetaKeyFromInsTags(pCxt);
}
return code; return code;
} }
......
source/libs/parser/src/parTranslater.c
浏览文件 @ 05a39977
...@@ -2181,7 +2181,11 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge ...@@ -2181,7 +2181,11 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge
} }
if (TSDB_DB_NAME_T == pTargetName->type) { if (TSDB_DB_NAME_T == pTargetName->type) {
return getDBVgInfoImpl(pCxt, pTargetName, pVgroupList); int32_t code = getDBVgInfoImpl(pCxt, pTargetName, pVgroupList);
if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
code = TSDB_CODE_SUCCESS;
}
return code;
} }
SVgroupInfo vgInfo = {0}; SVgroupInfo vgInfo = {0};
...@@ -2190,86 +2194,22 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge ...@@ -2190,86 +2194,22 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge
*pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo)); *pVgroupList = taosArrayInit(1, sizeof(SVgroupInfo));
if (NULL == *pVgroupList) { if (NULL == *pVgroupList) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
} else {
taosArrayPush(*pVgroupList, &vgInfo);
} }
} } else if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
if (TSDB_CODE_SUCCESS == code) { code = TSDB_CODE_SUCCESS;
taosArrayPush(*pVgroupList, &vgInfo);
} }
return code; return code;
} }
static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) {
if (OP_TYPE_EQUAL != pOper->opType) {
return TSDB_CODE_SUCCESS;
}
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
pVal = (SValueNode*)pOper->pLeft;
}
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight;
}
if (NULL == pCol || NULL == pVal) {
return TSDB_CODE_SUCCESS;
}
if (0 == strcmp(pCol->colName, "db_name")) {
return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal));
} else if (0 == strcmp(pCol->colName, "table_name")) {
return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal));
}
return TSDB_CODE_SUCCESS;
}
static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName);
}
}
static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) {
if (LOGIC_COND_TYPE_AND != pCond->condType) {
return TSDB_CODE_SUCCESS;
}
SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); }
if ('\0' == pName->dbname[0]) {
pName->type = 0;
}
return TSDB_CODE_SUCCESS;
}
static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_OPERATOR == nodeType(pWhere)) {
return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) {
return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName);
}
return TSDB_CODE_SUCCESS;
}
static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) {
if (!isSelectStmt(pCxt->pCurrStmt)) { if (!isSelectStmt(pCxt->pCurrStmt)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SName targetName = {0}; SName targetName = {0};
int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList);
} }
......
source/libs/parser/src/parUtil.c
浏览文件 @ 05a39977
...@@ -420,6 +420,75 @@ end: ...@@ -420,6 +420,75 @@ end:
return retCode; return retCode;
} }
static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pOper, SName* pName) {
if (OP_TYPE_EQUAL != pOper->opType) {
return TSDB_CODE_SUCCESS;
}
SColumnNode* pCol = NULL;
SValueNode* pVal = NULL;
if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) {
pCol = (SColumnNode*)pOper->pLeft;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) {
pVal = (SValueNode*)pOper->pLeft;
}
if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) {
pCol = (SColumnNode*)pOper->pRight;
} else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) {
pVal = (SValueNode*)pOper->pRight;
}
if (NULL == pCol || NULL == pVal) {
return TSDB_CODE_SUCCESS;
}
if (0 == strcmp(pCol->colName, "db_name")) {
return tNameSetDbName(pName, acctId, pVal->literal, strlen(pVal->literal));
} else if (0 == strcmp(pCol->colName, "table_name")) {
return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal));
}
return TSDB_CODE_SUCCESS;
}
static void getInsTagsTableTargetObjName(int32_t acctId, SNode* pNode, SName* pName) {
if (QUERY_NODE_OPERATOR == nodeType(pNode)) {
getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pNode, pName);
}
}
static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicConditionNode* pCond, SName* pName) {
if (LOGIC_COND_TYPE_AND != pCond->condType) {
return TSDB_CODE_SUCCESS;
}
SNode* pNode = NULL;
FOREACH(pNode, pCond->pParameterList) { getInsTagsTableTargetObjName(acctId, pNode, pName); }
if ('\0' == pName->dbname[0]) {
pName->type = 0;
}
return TSDB_CODE_SUCCESS;
}
int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) {
if (NULL == pWhere) {
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_OPERATOR == nodeType(pWhere)) {
int32_t code = getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pWhere, pName);
if (TSDB_CODE_SUCCESS == code && '\0' == pName->dbname[0]) {
pName->type = 0;
}
return code;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) {
return getInsTagsTableTargetNameFromCond(acctId, (SLogicConditionNode*)pWhere, pName);
}
return TSDB_CODE_SUCCESS;
}
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) {
return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type);
} }
...@@ -1173,7 +1242,7 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { ...@@ -1173,7 +1242,7 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) {
taosArrayDestroy(p->pTableVgroupReq); taosArrayDestroy(p->pTableVgroupReq);
p = taosHashIterate(pMetaCache->pInsertTables, p); p = taosHashIterate(pMetaCache->pInsertTables, p);
} }
taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pInsertTables);
taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbVgroup);
taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbCfg);
......
source/libs/parser/test/parSelectTest.cpp
浏览文件 @ 05a39977
...@@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) { ...@@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) {
} }
TEST_F(ParserSelectTest, informationSchema) { TEST_F(ParserSelectTest, informationSchema) {
useDb("root", "test"); useDb("root", "information_schema");
run("SELECT * FROM ins_databases WHERE name = 'information_schema'");
run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'");
} }
TEST_F(ParserSelectTest, withoutFrom) { TEST_F(ParserSelectTest, withoutFrom) {
......
source/libs/parser/test/parTestMain.cpp
浏览文件 @ 05a39977
...@@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment { ...@@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment {
private: private:
void initLog(const char* path) { void initLog(const char* path) {
int32_t logLevel = getLogLevel(); int32_t logLevel = getLogLevel() | DEBUG_SCREEN;
dDebugFlag = logLevel; dDebugFlag = logLevel;
vDebugFlag = logLevel; vDebugFlag = logLevel;
mDebugFlag = logLevel; mDebugFlag = logLevel;
......
source/libs/planner/test/planTestMain.cpp
浏览文件 @ 05a39977
...@@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment { ...@@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment {
private: private:
void initLog(const char* path) { void initLog(const char* path) {
int32_t logLevel = getLogLevel(); int32_t logLevel = getLogLevel() | DEBUG_SCREEN;
dDebugFlag = logLevel; dDebugFlag = logLevel;
vDebugFlag = logLevel; vDebugFlag = logLevel;
mDebugFlag = logLevel; mDebugFlag = logLevel;
......
source/libs/transport/src/transCli.c
浏览文件 @ 05a39977
...@@ -1347,7 +1347,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { ...@@ -1347,7 +1347,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
EPSET_FORWARD_INUSE(&pCtx->epSet); EPSET_FORWARD_INUSE(&pCtx->epSet);
} else { } else {
if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet) < 0) { if (tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet) < 0) {
tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn)); tError("%s conn %p failed to deserialize epset", CONN_GET_INST_LABEL(pConn), pConn);
} }
} }
addConnToPool(pThrd->pool, pConn); addConnToPool(pThrd->pool, pConn);
......
source/libs/transport/src/transComm.c
浏览文件 @ 05a39977
...@@ -509,7 +509,7 @@ void transDQCancel(SDelayQueue* queue, SDelayTask* task) { ...@@ -509,7 +509,7 @@ void transDQCancel(SDelayQueue* queue, SDelayTask* task) {
if (heapSize(queue->heap) != 0) { if (heapSize(queue->heap) != 0) {
HeapNode* minNode = heapMin(queue->heap); HeapNode* minNode = heapMin(queue->heap);
if (minNode != NULL) return; if (minNode == NULL) return;
uint64_t now = taosGetTimestampMs(); uint64_t now = taosGetTimestampMs();
SDelayTask* task = container_of(minNode, SDelayTask, node); SDelayTask* task = container_of(minNode, SDelayTask, node);
......
source/libs/transport/src/transSvr.c
浏览文件 @ 05a39977
...@@ -284,7 +284,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { ...@@ -284,7 +284,7 @@ void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
} }
return; return;
} else { } else {
tError("%s conn %p read invalid packet, exceed limit, received from %s, local info:", transLabel(pTransInst), tError("%s conn %p read invalid packet, exceed limit, received from %s, local info:%s", transLabel(pTransInst),
conn, conn->dst, conn->src); conn, conn->dst, conn->src);
destroyConn(conn, true); destroyConn(conn, true);
return; return;
...@@ -952,10 +952,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, ...@@ -952,10 +952,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
#ifdef WINDOWS #ifdef WINDOWS
char pipeName[64]; char pipeName[64];
snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%p-" PRIu64, taosSafeRand(), GetCurrentProcessId()); snprintf(pipeName, sizeof(pipeName), "\\\\?\\pipe\\trans.rpc.%d-%" PRIu64, taosSafeRand(), GetCurrentProcessId());
#else #else
char pipeName[PATH_MAX] = {0}; char pipeName[PATH_MAX] = {0};
snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08X-" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(), snprintf(pipeName, sizeof(pipeName), "%s%spipe.trans.rpc.%08d-%" PRIu64, tsTempDir, TD_DIRSEP, taosSafeRand(),
taosGetSelfPthreadId()); taosGetSelfPthreadId());
#endif #endif
ret = uv_pipe_bind(&srv->pipeListen, pipeName); ret = uv_pipe_bind(&srv->pipeListen, pipeName);
......
source/os/src/osSocket.c
浏览文件 @ 05a39977
...@@ -801,7 +801,6 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { ...@@ -801,7 +801,6 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) {
if (taosKeepTcpAlive(pSocket) < 0) { if (taosKeepTcpAlive(pSocket) < 0) {
// printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno)); // printf("failed to set tcp server keep-alive option, 0x%x:%hu(%s)", ip, port, strerror(errno));
taosCloseSocket(&pSocket);
return NULL; return NULL;
} }
......
source/util/src/tcompression.c
浏览文件 @ 05a39977
...@@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1; ...@@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1;
#define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL)
#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
#ifdef TD_TSZ #ifdef TD_TSZ
bool lossyFloat = false; bool lossyFloat = false;
...@@ -999,41 +999,212 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co ...@@ -999,41 +999,212 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
/************************************************************************* /*************************************************************************
* STREAM COMPRESSION * STREAM COMPRESSION
*************************************************************************/ *************************************************************************/
#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (a)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
typedef struct SCompressor SCompressor;
static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData);
static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; }
static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; }
static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; }
static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; }
static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; }
static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; }
static struct { static struct {
int8_t type; int8_t type;
int32_t bytes; int32_t bytes;
int8_t isVarLen; int8_t isVarLen;
int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg);
int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData);
int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *);
int64_t (*getI64)(const void *pData);
void (*putI64)(int64_t v, void *pData);
} DATA_TYPE_INFO[] = { } DATA_TYPE_INFO[] = {
{TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL {.type = TSDB_DATA_TYPE_NULL,
{TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL .bytes = 0,
{TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT .isVarLen = 0,
{TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT .startFn = NULL,
{TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT .cmprFn = NULL,
{TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT .endFn = NULL,
{TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT .getI64 = NULL,
{TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE .putI64 = NULL},
{TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR {.type = TSDB_DATA_TYPE_BOOL,
{TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP .bytes = 1,
{TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR .isVarLen = 0,
{TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT .startFn = tCompBoolStart,
{TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT .cmprFn = tCompBool,
{TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT .endFn = tCompBoolEnd,
{TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT .getI64 = NULL,
{TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON .putI64 = NULL},
{TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY {.type = TSDB_DATA_TYPE_TINYINT,
{TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL .bytes = 1,
{TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB .isVarLen = 0,
{TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB .startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI8,
.putI64 = tPutI64OfI8},
{.type = TSDB_DATA_TYPE_SMALLINT,
.bytes = 2,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI16,
.putI64 = tPutI64OfI16},
{.type = TSDB_DATA_TYPE_INT,
.bytes = 4,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI32,
.putI64 = tPutI64OfI32},
{.type = TSDB_DATA_TYPE_BIGINT,
.bytes = 8,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI64,
.putI64 = tPutI64OfI64},
{.type = TSDB_DATA_TYPE_FLOAT,
.bytes = 4,
.isVarLen = 0,
.startFn = tCompFloatStart,
.cmprFn = tCompFloat,
.endFn = tCompFloatEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_DOUBLE,
.bytes = 8,
.isVarLen = 0,
.startFn = tCompDoubleStart,
.cmprFn = tCompDouble,
.endFn = tCompDoubleEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_VARCHAR,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_TIMESTAMP,
.bytes = 8,
.isVarLen = 0,
.startFn = tCompTimestampStart,
.cmprFn = tCompTimestamp,
.endFn = tCompTimestampEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_NCHAR,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_UTINYINT,
.bytes = 1,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI8,
.putI64 = tPutI64OfI8},
{.type = TSDB_DATA_TYPE_USMALLINT,
.bytes = 2,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI16,
.putI64 = tPutI64OfI16},
{.type = TSDB_DATA_TYPE_UINT,
.bytes = 4,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI32,
.putI64 = tPutI64OfI32},
{.type = TSDB_DATA_TYPE_UBIGINT,
.bytes = 8,
.isVarLen = 0,
.startFn = tCompIntStart,
.cmprFn = tCompInt,
.endFn = tCompIntEnd,
.getI64 = tGetI64OfI64,
.putI64 = tPutI64OfI64},
{.type = TSDB_DATA_TYPE_JSON,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_VARBINARY,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_DECIMAL,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_BLOB,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
{.type = TSDB_DATA_TYPE_MEDIUMBLOB,
.bytes = 1,
.isVarLen = 1,
.startFn = tCompBinaryStart,
.cmprFn = tCompBinary,
.endFn = tCompBinaryEnd,
.getI64 = NULL,
.putI64 = NULL},
}; };
struct SCompressor { struct SCompressor {
...@@ -1041,8 +1212,9 @@ struct SCompressor { ...@@ -1041,8 +1212,9 @@ struct SCompressor {
int8_t cmprAlg; int8_t cmprAlg;
int8_t autoAlloc; int8_t autoAlloc;
int32_t nVal; int32_t nVal;
uint8_t *aBuf[2]; uint8_t *pBuf;
int64_t nBuf[2]; int32_t nBuf;
uint8_t *aBuf[1];
union { union {
// Timestamp ---- // Timestamp ----
struct { struct {
...@@ -1056,6 +1228,7 @@ struct SCompressor { ...@@ -1056,6 +1228,7 @@ struct SCompressor {
int32_t i_selector; int32_t i_selector;
int32_t i_start; int32_t i_start;
int32_t i_end; int32_t i_end;
int32_t i_nEle;
uint64_t i_aZigzag[241]; uint64_t i_aZigzag[241];
int8_t i_aBitN[241]; int8_t i_aBitN[241];
}; };
...@@ -1072,86 +1245,114 @@ struct SCompressor { ...@@ -1072,86 +1245,114 @@ struct SCompressor {
}; };
}; };
static int32_t tTwoStageComp(SCompressor *pCmprsor, int32_t *szComp) {
int32_t code = 0;
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) {
return code;
}
*szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf);
if (*szComp && *szComp < pCmprsor->nBuf) {
pCmprsor->aBuf[0][0] = 1;
*szComp += 1;
} else {
pCmprsor->aBuf[0][0] = 0;
memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf);
*szComp = pCmprsor->nBuf + 1;
}
return code;
}
// Timestamp ===================================================== // Timestamp =====================================================
static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0; int32_t code = 0;
if (pCmprsor->nVal) { pCmprsor->nBuf = 1;
if (pCmprsor->autoAlloc) {
code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal);
if (code) return code;
}
pCmprsor->nBuf[1] = 0;
int64_t n = 1;
int64_t value;
int64_t delta;
uint64_t vZigzag;
while (n < pCmprsor->nBuf[0]) {
uint8_t aN[2];
aN[0] = pCmprsor->aBuf[0][n] & 0xf;
aN[1] = pCmprsor->aBuf[0][n] >> 4;
n++;
for (int32_t i = 0; i < 2; i++) {
vZigzag = 0;
for (uint8_t j = 0; j < aN[i]; j++) {
vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j));
n++;
}
int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
if (pCmprsor->nBuf[1] == 0) { if (code) return code;
delta = 0;
value = delta_of_delta;
} else {
delta = delta_of_delta + delta;
value = delta + value;
}
memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t)); pCmprsor->pBuf[0] = 1;
pCmprsor->nBuf[1] += sizeof(int64_t);
if (n >= pCmprsor->nBuf[0]) break; return code;
}
static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) {
int32_t code = 0;
if (pCmprsor->nVal == 0) goto _exit;
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(int64_t) * pCmprsor->nVal + 1))) {
return code;
}
int32_t n = 1;
int32_t nBuf = 1;
int64_t value;
int64_t delta;
for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
uint8_t aN[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
n++;
for (int32_t i = 0; i < 2; i++) {
uint64_t vZigzag = 0;
for (uint8_t j = 0; j < aN[i]; j++) {
vZigzag |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j));
n++;
} }
}
ASSERT(n == pCmprsor->nBuf[0]); int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
if (iVal) {
delta = delta_of_delta + delta;
value = delta + value;
} else {
delta = 0;
value = delta_of_delta;
}
if (pCmprsor->autoAlloc) { memcpy(pCmprsor->aBuf[0] + nBuf, &value, sizeof(value));
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1); nBuf += sizeof(int64_t);
if (code) return code;
iVal++;
if (iVal >= pCmprsor->nVal) break;
} }
memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]);
pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1];
} }
pCmprsor->aBuf[0][0] = 0;
ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nVal + 1);
uint8_t *pBuf = pCmprsor->pBuf;
pCmprsor->pBuf = pCmprsor->aBuf[0];
pCmprsor->aBuf[0] = pBuf;
pCmprsor->nBuf = nBuf;
_exit:
pCmprsor->pBuf[0] = 0;
return code; return code;
} }
static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) { static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0; int32_t code = 0;
int64_t ts = *(int64_t *)pData; int64_t ts = *(int64_t *)pData;
ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(nData == 8); ASSERT(nData == 8);
if (pCmprsor->aBuf[0][0] == 1) { if (pCmprsor->pBuf[0] == 1) {
if (pCmprsor->nVal == 0) { if (pCmprsor->nVal == 0) {
pCmprsor->ts_prev_val = ts; pCmprsor->ts_prev_val = ts;
pCmprsor->ts_prev_delta = -ts; pCmprsor->ts_prev_delta = -ts;
} }
if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
code = tCompSetCopyMode(pCmprsor); code = tCompTSSwitchToCopy(pCmprsor);
if (code) return code; if (code) return code;
goto _copy_cmpr; goto _copy_cmpr;
} }
int64_t delta = ts - pCmprsor->ts_prev_val; int64_t delta = ts - pCmprsor->ts_prev_val;
if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
code = tCompSetCopyMode(pCmprsor); code = tCompTSSwitchToCopy(pCmprsor);
if (code) return code; if (code) return code;
goto _copy_cmpr; goto _copy_cmpr;
} }
...@@ -1162,43 +1363,63 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t ...@@ -1162,43 +1363,63 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t
pCmprsor->ts_prev_delta = delta; pCmprsor->ts_prev_delta = delta;
if ((pCmprsor->nVal & 0x1) == 0) { if ((pCmprsor->nVal & 0x1) == 0) {
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); return code;
if (code) return code;
} }
pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0]; pCmprsor->ts_flag_p = pCmprsor->pBuf + pCmprsor->nBuf;
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0] = 0; pCmprsor->ts_flag_p[0] = 0;
while (vZigzag) { while (vZigzag) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0]++; pCmprsor->ts_flag_p[0]++;
vZigzag >>= 8; vZigzag >>= 8;
} }
} else { } else {
while (vZigzag) { while (vZigzag) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0] += 0x10; pCmprsor->ts_flag_p[0] += 0x10;
vZigzag >>= 8; vZigzag >>= 8;
} }
} }
} else { } else {
_copy_cmpr: _copy_cmpr:
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(ts)))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts)); return code;
if (code) return code;
} }
memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts)); memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &ts, sizeof(ts));
pCmprsor->nBuf[0] += sizeof(ts); pCmprsor->nBuf += sizeof(ts);
} }
pCmprsor->nVal++; pCmprsor->nVal++;
return code; return code;
} }
static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) {
code = tCompTSSwitchToCopy(pCmprsor);
if (code) return code;
}
if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
code = tTwoStageComp(pCmprsor, nData);
if (code) return code;
*ppData = pCmprsor->aBuf[0];
} else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
} else {
ASSERT(0);
}
return code;
}
// Integer ===================================================== // Integer =====================================================
#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
...@@ -1207,94 +1428,163 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 ...@@ -1207,94 +1428,163 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10
13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
static const int32_t NEXT_IDX[] = {
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66,
67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154,
155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176,
177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198,
199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,
221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 0};
static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0;
static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) { pCmprsor->i_prev = 0;
pCmprsor->i_selector = 0;
pCmprsor->i_start = 0;
pCmprsor->i_end = 0;
pCmprsor->i_nEle = 0;
pCmprsor->nBuf = 1;
code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
if (code) return code;
pCmprsor->pBuf[0] = 0;
return code;
}
static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) {
int32_t code = 0; int32_t code = 0;
ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); if (pCmprsor->nVal == 0) goto _exit;
if (pCmprsor->aBuf[0][0] == 0) { int32_t size = DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1;
int64_t val; if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], size))) {
return code;
}
switch (pCmprsor->type) { int32_t n = 1;
case TSDB_DATA_TYPE_TINYINT: int32_t nBuf = 1;
val = *(int8_t *)pData; int64_t vPrev = 0;
break; while (n < pCmprsor->nBuf) {
case TSDB_DATA_TYPE_SMALLINT: uint64_t b;
val = *(int16_t *)pData; memcpy(&b, pCmprsor->pBuf + n, sizeof(b));
break; n += sizeof(b);
case TSDB_DATA_TYPE_INT:
val = *(int32_t *)pData; int32_t i_selector = (b & 0xf);
break; int32_t nEle = SELECTOR_TO_ELEMS[i_selector];
case TSDB_DATA_TYPE_BIGINT: uint8_t bits = BIT_PER_INTEGER[i_selector];
val = *(int64_t *)pData; uint64_t mask = (((uint64_t)1) << bits) - 1;
break; for (int32_t iEle = 0; iEle < nEle; iEle++) {
case TSDB_DATA_TYPE_UTINYINT: uint64_t vZigzag = (b >> (bits * iEle + 4)) & mask;
val = *(uint8_t *)pData; vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev;
break;
case TSDB_DATA_TYPE_USMALLINT: DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf);
val = *(uint16_t *)pData; nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes;
break;
case TSDB_DATA_TYPE_UINT:
val = *(uint32_t *)pData;
break;
case TSDB_DATA_TYPE_UBIGINT:
val = *(int64_t *)pData;
break;
default:
ASSERT(0);
break;
} }
}
while (pCmprsor->i_nEle) {
vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev;
memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes);
nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes;
pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
pCmprsor->i_nEle--;
}
ASSERT(n == pCmprsor->nBuf && nBuf == size);
uint8_t *pBuf = pCmprsor->pBuf;
pCmprsor->pBuf = pCmprsor->aBuf[0];
pCmprsor->aBuf[0] = pBuf;
pCmprsor->nBuf = size;
_exit:
pCmprsor->pBuf[0] = 1;
return code;
}
static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes);
if (pCmprsor->pBuf[0] == 0) {
int64_t val = DATA_TYPE_INFO[pCmprsor->type].getI64(pData);
if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) {
// TODO code = tCompIntSwitchToCopy(pCmprsor);
if (code) return code;
goto _copy_cmpr; goto _copy_cmpr;
} }
int64_t diff = val - pCmprsor->i_prev; int64_t diff = val - pCmprsor->i_prev;
uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff); uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
if (vZigzag >= SIMPLE8B_MAX) { if (vZigzag >= SIMPLE8B_MAX) {
// TODO code = tCompIntSwitchToCopy(pCmprsor);
if (code) return code;
goto _copy_cmpr; goto _copy_cmpr;
} }
int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0; int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0;
pCmprsor->i_prev = val; pCmprsor->i_prev = val;
while (1) { for (;;) {
int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; if (pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] &&
pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) {
pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; pCmprsor->i_selector = BIT_TO_SELECTOR[nBit];
} }
pCmprsor->i_end = (pCmprsor->i_end + 1) % 241;
pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag;
pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit;
pCmprsor->i_end = NEXT_IDX[pCmprsor->i_end];
pCmprsor->i_nEle++;
break; break;
} else { } else {
while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
pCmprsor->i_selector++; int32_t lidx = pCmprsor->i_selector + 1;
int32_t ridx = 15;
while (lidx <= ridx) {
pCmprsor->i_selector = (lidx + ridx) >> 1;
if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
lidx = pCmprsor->i_selector + 1;
} else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
ridx = pCmprsor->i_selector - 1;
} else {
break;
}
}
if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
} }
nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t)); return code;
if (code) return code;
} }
uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]); uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf);
pCmprsor->nBuf[0] += sizeof(uint64_t); pCmprsor->nBuf += sizeof(uint64_t);
bp[0] = pCmprsor->i_selector; bp[0] = pCmprsor->i_selector;
uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector];
for (int32_t iVal = 0; iVal < nEle; iVal++) { for (int32_t iVal = 0; iVal < nEle; iVal++) {
bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4));
pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
pCmprsor->i_nEle--;
} }
// reset and continue // reset and continue
pCmprsor->i_selector = 0; pCmprsor->i_selector = 0;
for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) { for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = NEXT_IDX[iVal]) {
if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) {
pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]; pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]];
} }
...@@ -1303,18 +1593,146 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ...@@ -1303,18 +1593,146 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
} }
} else { } else {
_copy_cmpr: _copy_cmpr:
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData);
if (code) return code; if (code) return code;
memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData);
pCmprsor->nBuf[0] += nData; pCmprsor->nBuf += nData;
} }
pCmprsor->nVal++; pCmprsor->nVal++;
return code; return code;
} }
static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
for (; pCmprsor->i_nEle;) {
if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
int32_t lidx = pCmprsor->i_selector + 1;
int32_t ridx = 15;
while (lidx <= ridx) {
pCmprsor->i_selector = (lidx + ridx) >> 1;
if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
lidx = pCmprsor->i_selector + 1;
} else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
ridx = pCmprsor->i_selector - 1;
} else {
break;
}
}
if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
}
int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) {
return code;
}
uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf);
pCmprsor->nBuf += sizeof(uint64_t);
bp[0] = pCmprsor->i_selector;
uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector];
for (int32_t iVal = 0; iVal < nEle; iVal++) {
bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4));
pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
pCmprsor->i_nEle--;
}
pCmprsor->i_selector = 0;
}
if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) {
code = tCompIntSwitchToCopy(pCmprsor);
if (code) return code;
}
if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
code = tTwoStageComp(pCmprsor, nData);
if (code) return code;
*ppData = pCmprsor->aBuf[0];
} else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
} else {
ASSERT(0);
}
return code;
}
// Float ===================================================== // Float =====================================================
static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0;
pCmprsor->f_prev = 0;
pCmprsor->f_flag_p = NULL;
pCmprsor->nBuf = 1;
code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
if (code) return code;
pCmprsor->pBuf[0] = 0;
return code;
}
static int32_t tCompFloatSwitchToCopy(SCompressor *pCmprsor) {
int32_t code = 0;
if (pCmprsor->nVal == 0) goto _exit;
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(float) * pCmprsor->nVal + 1))) {
return code;
}
int32_t n = 1;
int32_t nBuf = 1;
union {
float f;
uint32_t u;
} val = {.u = 0};
for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
n++;
for (int8_t i = 0; i < 2; i++) {
uint8_t flag = flags[i];
uint32_t diff = 0;
int8_t nBytes = (flag & 0x7) + 1;
for (int j = 0; j < nBytes; j++) {
diff |= (((uint32_t)pCmprsor->pBuf[n]) << (8 * j));
n++;
}
if (flag & 0x8) {
diff <<= (32 - nBytes * 8);
}
val.u ^= diff;
memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val));
nBuf += sizeof(val);
iVal++;
if (iVal >= pCmprsor->nVal) break;
}
}
uint8_t *pBuf = pCmprsor->pBuf;
pCmprsor->pBuf = pCmprsor->aBuf[0];
pCmprsor->aBuf[0] = pBuf;
pCmprsor->nBuf = nBuf;
_exit:
pCmprsor->pBuf[0] = 1;
return code;
}
static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0; int32_t code = 0;
...@@ -1347,13 +1765,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat ...@@ -1347,13 +1765,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
if (nBytes == 0) nBytes++; if (nBytes == 0) nBytes++;
if ((pCmprsor->nVal & 0x1) == 0) { if ((pCmprsor->nVal & 0x1) == 0) {
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 9))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9); return code;
if (code) return code;
} }
pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; pCmprsor->f_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf];
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
if (clz < ctz) { if (clz < ctz) {
pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
...@@ -1368,8 +1785,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat ...@@ -1368,8 +1785,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
} }
} }
for (; nBytes; nBytes--) { for (; nBytes; nBytes--) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
diff >>= BITS_PER_BYTE; diff >>= BITS_PER_BYTE;
} }
pCmprsor->nVal++; pCmprsor->nVal++;
...@@ -1377,7 +1794,98 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat ...@@ -1377,7 +1794,98 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
return code; return code;
} }
static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
if (pCmprsor->nBuf >= sizeof(float) * pCmprsor->nVal + 1) {
code = tCompFloatSwitchToCopy(pCmprsor);
if (code) return code;
}
if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
code = tTwoStageComp(pCmprsor, nData);
if (code) return code;
*ppData = pCmprsor->aBuf[0];
} else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
} else {
ASSERT(0);
}
return code;
}
// Double ===================================================== // Double =====================================================
static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0;
pCmprsor->d_prev = 0;
pCmprsor->d_flag_p = NULL;
pCmprsor->nBuf = 1;
code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
if (code) return code;
pCmprsor->pBuf[0] = 0;
return code;
}
static int32_t tCompDoubleSwitchToCopy(SCompressor *pCmprsor) {
int32_t code = 0;
if (pCmprsor->nVal == 0) goto _exit;
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(double) * pCmprsor->nVal + 1))) {
return code;
}
int32_t n = 1;
int32_t nBuf = 1;
union {
double f;
uint64_t u;
} val = {.u = 0};
for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
n++;
for (int8_t i = 0; i < 2; i++) {
uint8_t flag = flags[i];
uint64_t diff = 0;
int8_t nBytes = (flag & 0x7) + 1;
for (int j = 0; j < nBytes; j++) {
diff |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j));
n++;
}
if (flag & 0x8) {
diff <<= (64 - nBytes * 8);
}
val.u ^= diff;
memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val));
nBuf += sizeof(val);
iVal++;
if (iVal >= pCmprsor->nVal) break;
}
}
uint8_t *pBuf = pCmprsor->pBuf;
pCmprsor->pBuf = pCmprsor->aBuf[0];
pCmprsor->aBuf[0] = pBuf;
pCmprsor->nBuf = nBuf;
_exit:
pCmprsor->pBuf[0] = 1;
return code;
}
static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0; int32_t code = 0;
...@@ -1410,13 +1918,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa ...@@ -1410,13 +1918,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
if (nBytes == 0) nBytes++; if (nBytes == 0) nBytes++;
if ((pCmprsor->nVal & 0x1) == 0) { if ((pCmprsor->nVal & 0x1) == 0) {
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); return code;
if (code) return code;
} }
pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; pCmprsor->d_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf];
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
if (clz < ctz) { if (clz < ctz) {
pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
...@@ -1431,8 +1938,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa ...@@ -1431,8 +1938,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
} }
} }
for (; nBytes; nBytes--) { for (; nBytes; nBytes--) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff);
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
diff >>= BITS_PER_BYTE; diff >>= BITS_PER_BYTE;
} }
pCmprsor->nVal++; pCmprsor->nVal++;
...@@ -1440,51 +1947,122 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa ...@@ -1440,51 +1947,122 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
return code; return code;
} }
static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
if (pCmprsor->nBuf >= sizeof(double) * pCmprsor->nVal + 1) {
code = tCompDoubleSwitchToCopy(pCmprsor);
if (code) return code;
}
if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
code = tTwoStageComp(pCmprsor, nData);
if (code) return code;
*ppData = pCmprsor->aBuf[0];
} else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
} else {
ASSERT(0);
}
return code;
}
// Binary ===================================================== // Binary =====================================================
static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
pCmprsor->nBuf = 1;
return 0;
}
static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0; int32_t code = 0;
if (nData) { if (nData) {
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); return code;
if (code) return code;
} }
memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData);
pCmprsor->nBuf[0] += nData; pCmprsor->nBuf += nData;
} }
pCmprsor->nVal++; pCmprsor->nVal++;
return code; return code;
} }
static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
if (pCmprsor->nBuf == 1) return code;
if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf))) {
return code;
}
int32_t szComp =
LZ4_compress_default(pCmprsor->pBuf + 1, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf - 1, pCmprsor->nBuf - 1);
if (szComp && szComp < pCmprsor->nBuf - 1) {
pCmprsor->aBuf[0][0] = 1;
*ppData = pCmprsor->aBuf[0];
*nData = szComp + 1;
} else {
pCmprsor->pBuf[0] = 0;
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
}
return code;
}
// Bool ===================================================== // Bool =====================================================
static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
pCmprsor->nBuf = 0;
return 0;
}
static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0; int32_t code = 0;
bool vBool = *(int8_t *)pData; bool vBool = *(int8_t *)pData;
int32_t mod4 = pCmprsor->nVal & 3; int32_t mod4 = (pCmprsor->nVal & 3);
if (mod4 == 0) { if (mod4 == 0) {
pCmprsor->nBuf[0]++; pCmprsor->nBuf++;
if (pCmprsor->autoAlloc) { if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf))) {
code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]); return code;
if (code) return code;
} }
pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0; pCmprsor->pBuf[pCmprsor->nBuf - 1] = 0;
} }
if (vBool) { if (vBool) {
pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4]; pCmprsor->pBuf[pCmprsor->nBuf - 1] |= BOOL_CMPR_TABLE[mod4];
} }
pCmprsor->nVal++; pCmprsor->nVal++;
return code; return code;
} }
static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
int32_t code = 0;
if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
code = tTwoStageComp(pCmprsor, nData);
if (code) return code;
*ppData = pCmprsor->aBuf[0];
} else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
*ppData = pCmprsor->pBuf;
*nData = pCmprsor->nBuf;
} else {
ASSERT(0);
}
return code;
}
// SCompressor ===================================================== // SCompressor =====================================================
int32_t tCompressorCreate(SCompressor **ppCmprsor) { int32_t tCompressorCreate(SCompressor **ppCmprsor) {
int32_t code = 0; int32_t code = 0;
...@@ -1492,124 +2070,334 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) { ...@@ -1492,124 +2070,334 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) {
*ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor)); *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor));
if ((*ppCmprsor) == NULL) { if ((*ppCmprsor) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; return code;
}
code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024);
if (code) {
taosMemoryFree(*ppCmprsor);
*ppCmprsor = NULL;
goto _exit;
} }
_exit:
return code; return code;
} }
int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t tCompressorDestroy(SCompressor *pCmprsor) {
int32_t code = 0; int32_t code = 0;
if (pCmprsor) { tFree(pCmprsor->pBuf);
int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
tFree(pCmprsor->aBuf[iBuf]);
}
taosMemoryFree(pCmprsor); int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
tFree(pCmprsor->aBuf[iBuf]);
} }
taosMemoryFree(pCmprsor);
return code; return code;
} }
int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) { int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0; int32_t code = 0;
pCmprsor->type = type; pCmprsor->type = type;
pCmprsor->cmprAlg = cmprAlg; pCmprsor->cmprAlg = cmprAlg;
pCmprsor->autoAlloc = autoAlloc; pCmprsor->autoAlloc = 1;
pCmprsor->nVal = 0; pCmprsor->nVal = 0;
switch (type) { if (DATA_TYPE_INFO[type].startFn) {
case TSDB_DATA_TYPE_TIMESTAMP: DATA_TYPE_INFO[type].startFn(pCmprsor, type, cmprAlg);
pCmprsor->ts_prev_val = 0;
pCmprsor->ts_prev_delta = 0;
pCmprsor->ts_flag_p = NULL;
pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise
pCmprsor->nBuf[0] = 1;
break;
case TSDB_DATA_TYPE_BOOL:
pCmprsor->nBuf[0] = 0;
break;
case TSDB_DATA_TYPE_BINARY:
pCmprsor->nBuf[0] = 0;
break;
case TSDB_DATA_TYPE_FLOAT:
pCmprsor->f_prev = 0;
pCmprsor->f_flag_p = NULL;
pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
pCmprsor->nBuf[0] = 1;
break;
case TSDB_DATA_TYPE_DOUBLE:
pCmprsor->d_prev = 0;
pCmprsor->d_flag_p = NULL;
pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
pCmprsor->nBuf[0] = 1;
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
pCmprsor->i_prev = 0;
pCmprsor->i_selector = 0;
pCmprsor->i_start = 0;
pCmprsor->i_end = 0;
pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
pCmprsor->nBuf[0] = 1;
break;
default:
break;
} }
return code; return code;
} }
int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) { int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin) {
int32_t code = 0; int32_t code = 0;
if (pCmprsor->nVal == 0) { *ppOut = NULL;
*ppData = NULL; *nOut = 0;
*nData = 0; if (nOrigin) {
return code; if (DATA_TYPE_INFO[pCmprsor->type].isVarLen) {
*nOrigin = pCmprsor->nBuf - 1;
} else {
*nOrigin = pCmprsor->nVal * DATA_TYPE_INFO[pCmprsor->type].bytes;
}
} }
if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) { if (pCmprsor->nVal == 0) return code;
code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
if (code) return code; if (DATA_TYPE_INFO[pCmprsor->type].endFn) {
return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppOut, nOut);
}
return code;
}
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
}
/*************************************************************************
* REGULAR COMPRESSION
*************************************************************************/
// Timestamp =====================================================
int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressTimestampImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
void *pBuf, int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressTimestampImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressTimestampImp(pBuf, nEle, pOut);
} else {
assert(0);
return -1;
}
}
// Float =====================================================
int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
// lossy mode
if (lossyFloat) {
return tsCompressFloatLossyImp(pIn, nEle, pOut);
// lossless mode
} else {
#endif
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressFloatImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressFloatImp(pIn, nEle, pBuf);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
}
int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]); int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
if (ret) { int32_t nBuf) {
pCmprsor->aBuf[1][0] = 0; #ifdef TD_TSZ
pCmprsor->nBuf[1] = ret + 1; if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressFloatImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressFloatImp(pBuf, nEle, pOut);
} else { } else {
pCmprsor->aBuf[1][0] = 1; assert(0);
memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]); return -1;
pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1;
} }
#ifdef TD_TSZ
}
#endif
}
*ppData = pCmprsor->aBuf[1]; // Double =====================================================
*nData = pCmprsor->nBuf[1]; int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (lossyDouble) {
// lossy mode
return tsCompressDoubleLossyImp(pIn, nEle, pOut);
} else { } else {
*ppData = pCmprsor->aBuf[0]; #endif
*nData = pCmprsor->nBuf[0]; // lossless mode
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressDoubleImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
} }
#endif
}
return code; int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
#ifdef TD_TSZ
if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
// decompress lossy
return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
} else {
#endif
// decompress lossless
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressDoubleImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressDoubleImp(pBuf, nEle, pOut);
} else {
assert(0);
return -1;
}
#ifdef TD_TSZ
}
#endif
} }
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { // Binary =====================================================
return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
} int32_t nBuf) {
\ No newline at end of file return tsCompressStringImp(pIn, nIn, pOut, nOut);
}
int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
return tsDecompressStringImp(pIn, nIn, pOut, nOut);
}
// Bool =====================================================
int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressBoolImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressBoolImp(pIn, nEle, pBuf);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressBoolImp(pIn, nEle, pOut);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressBoolImp(pBuf, nEle, pOut);
} else {
assert(0);
return -1;
}
}
// Tinyint =====================================================
int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
} else {
assert(0);
return -1;
}
}
// Smallint =====================================================
int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
void *pBuf, int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
} else {
assert(0);
return -1;
}
}
// Int =====================================================
int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT);
} else {
assert(0);
return -1;
}
}
// Bigint =====================================================
int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT);
return tsCompressStringImp(pBuf, len, pOut, nOut);
} else {
assert(0);
return -1;
}
}
int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
int32_t nBuf) {
if (cmprAlg == ONE_STAGE_COMP) {
return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
} else if (cmprAlg == TWO_STAGE_COMP) {
if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
} else {
assert(0);
return -1;
}
}
tests/script/tsim/stream/sliding.sim
浏览文件 @ 05a39977
...@@ -369,9 +369,10 @@ endi ...@@ -369,9 +369,10 @@ endi
#$loop_all = 0 #$loop_all = 0
#=looptest: #=looptest:
sql drop database IF EXISTS test2;
sql drop stream IF EXISTS streams21; sql drop stream IF EXISTS streams21;
sql drop stream IF EXISTS streams22; sql drop stream IF EXISTS streams22;
sql drop stream IF EXISTS streams23;
sql drop database IF EXISTS test2;
sql create database test2 vgroups 6; sql create database test2 vgroups 6;
sql use test2; sql use test2;
...@@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2); ...@@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2);
sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s);
sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s);
sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s);
sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791213000,1,1,1,1.0);
sql insert into t1 values(1648791223001,2,2,2,1.1); sql insert into t1 values(1648791223001,2,2,2,1.1);
...@@ -508,6 +510,50 @@ if $data32 != 8 then ...@@ -508,6 +510,50 @@ if $data32 != 8 then
goto loop3 goto loop3
endi endi
$loop_count = 0
loop4:
sleep 100
$loop_count = $loop_count + 1
if $loop_count == 10 then
return -1
endi
sql select * from streamt3;
# row 0
if $rows != 5 then
print =====rows=$rows
goto loop4
endi
if $data01 != 4 then
print =====data01=$data01
goto loop4
endi
if $data11 != 6 then
print =====data11=$data11
goto loop4
endi
if $data21 != 4 then
print =====data21=$data21
goto loop4
endi
if $data31 != 4 then
print =====data31=$data31
goto loop4
endi
if $data41 != 2 then
print =====data41=$data41
goto loop4
endi
$loop_all = $loop_all + 1 $loop_all = $loop_all + 1
print ============loop_all=$loop_all print ============loop_all=$loop_all
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或