diff --git a/2.0/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h index 9c738dad988e4be8d223711a83892e7623d1b27a..970b8263035420ff4bc67e1b192dc7f0f6ffc5f8 100644 --- a/2.0/src/query/inc/qExecutor.h +++ b/2.0/src/query/inc/qExecutor.h @@ -589,7 +589,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRunt SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c index ca656b81ff0c7c269c86bf2613ada1983afd22ae..490584c75a5e5fed0ea590189cb22f465f66e06f 100644 --- a/2.0/src/query/src/qExecutor.c +++ b/2.0/src/query/src/qExecutor.c @@ -2166,7 +2166,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_TimeWindow: { pRuntimeEnv->proot = - createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + createIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; if (opType != OP_DummyInput && opType != OP_Join) { setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); @@ -6756,7 +6756,7 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index bae083cf9451632a5d1e2b57b8546b60634674d1..b28adfaaca4426873d1c12c250ae1c55bc002938 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -10,7 +10,7 @@ #include #include #include "raftServer.h" -#include "common.h" +#include "tcommon.h" const char *exe_name; diff --git a/contrib/test/craft/raftServer.c b/contrib/test/craft/raftServer.c index ffec22e6463b0cafcfd986d9980e3466a7c9c20d..c4f347f6f5ae61d7d1d2579f5492447a022a99a6 100644 --- a/contrib/test/craft/raftServer.c +++ b/contrib/test/craft/raftServer.c @@ -1,5 +1,5 @@ #include -#include "common.h" +#include "tcommon.h" #include "raftServer.h" char *keys; diff --git a/contrib/test/craft/raftServer.h b/contrib/test/craft/raftServer.h index f4087cf1a9b8d664a56c58bf797e01f60cba032a..fb6f9d299617b85dafdabc70fabf7fece6d1a17f 100644 --- a/contrib/test/craft/raftServer.h +++ b/contrib/test/craft/raftServer.h @@ -11,7 +11,7 @@ extern "C" { #include #include "raft.h" #include "raft/uv.h" -#include "common.h" +#include "tcommon.h" // simulate a db store, just for test diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e5a7a3563e1348b7450f84b936fcb95f7208dd0a..66cb0bb4ba5c48e6320455a94d5b6778c283fbe4 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -16,27 +16,19 @@ #ifndef _TD_COMMON_TAOS_DEF_H_ #define _TD_COMMON_TAOS_DEF_H_ +#include "taos.h" +#include "tdef.h" + #ifdef __cplusplus extern "C" { #endif -#include "taos.h" -#include "tdef.h" - typedef int64_t tb_uid_t; -#define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) -#define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) +#define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) -typedef enum { - TAOS_QTYPE_RPC = 1, - TAOS_QTYPE_FWD = 2, - TAOS_QTYPE_WAL = 3, - TAOS_QTYPE_CQ = 4, - TAOS_QTYPE_QUERY = 5 -} EQType; - typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table @@ -46,14 +38,6 @@ typedef enum { TSDB_TABLE_MAX = 6 } ETableType; -typedef enum { - TSDB_MOD_MNODE = 1, - TSDB_MOD_HTTP = 2, - TSDB_MOD_MONITOR = 3, - TSDB_MOD_MQTT = 4, - TSDB_MOD_MAX = 5 -} EModuleType; - typedef enum { TSDB_CHECK_ITEM_NETWORK, TSDB_CHECK_ITEM_MEM, diff --git a/include/common/common.h b/include/common/tcommon.h similarity index 98% rename from include/common/common.h rename to include/common/tcommon.h index 8fa2d03d6d3f79a220b6c10a38070a8a42026155..37d20cdb974b8f56b8f69b8ecba64815d882c7f7 100644 --- a/include/common/common.h +++ b/include/common/tcommon.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_H -#define TDENGINE_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_COMMON_DEF_H_ +#define _TD_COMMON_DEF_H_ #include "taosdef.h" #include "tarray.h" #include "tmsg.h" #include "tvariant.h" + +#ifdef __cplusplus +extern "C" { +#endif + // typedef struct STimeWindow { // TSKEY skey; // TSKEY ekey; @@ -214,7 +215,7 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); pRsp->pBlockData = NULL; - // for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + // for (int32_t i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); // tDeleteSSDataBlock(pDataBlock); //} @@ -279,4 +280,4 @@ typedef struct SSessionWindow { } #endif -#endif // TDENGINE_COMMON_H +#endif /*_TD_COMMON_DEF_H_*/ diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 4a73cf7dd21ceb733b04e8f5ba62bb83ebbc674c..89a0e6b4dc5adf1659fc2a582b64cc355b2c9c89 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + #ifndef _TD_COMMON_DATA_FORMAT_H_ #define _TD_COMMON_DATA_FORMAT_H_ @@ -65,40 +66,40 @@ typedef struct { uint16_t offset; // point offset in STpRow after the header part. } STColumn; -#define colType(col) ((col)->type) -#define colColId(col) ((col)->colId) -#define colBytes(col) ((col)->bytes) +#define colType(col) ((col)->type) +#define colColId(col) ((col)->colId) +#define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) -#define colSetType(col, t) (colType(col) = (t)) +#define colSetType(col, t) (colType(col) = (t)) #define colSetColId(col, id) (colColId(col) = (id)) -#define colSetBytes(col, b) (colBytes(col) = (b)) +#define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) // ----------------- TSDB SCHEMA DEFINITION typedef struct { - int version; // version - int numOfCols; // Number of columns appended - int tlen; // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + + int32_t version; // version + int32_t numOfCols; // Number of columns appended + int32_t tlen; // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + // (bytes)) uint16_t flen; // First part length in a STpRow after the header part uint16_t vlen; // pure value part length, excluded the overhead (bytes only) STColumn columns[]; } STSchema; -#define schemaNCols(s) ((s)->numOfCols) -#define schemaVersion(s) ((s)->version) -#define schemaTLen(s) ((s)->tlen) -#define schemaFLen(s) ((s)->flen) -#define schemaVLen(s) ((s)->vlen) +#define schemaNCols(s) ((s)->numOfCols) +#define schemaVersion(s) ((s)->version) +#define schemaTLen(s) ((s)->tlen) +#define schemaFLen(s) ((s)->flen) +#define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(const STSchema *pSchema); -int tdEncodeSchema(void **buf, STSchema *pSchema); -void * tdDecodeSchema(void *buf, STSchema **pRSchema); +int32_t tdEncodeSchema(void **buf, STSchema *pSchema); +void *tdDecodeSchema(void *buf, STSchema **pRSchema); -static FORCE_INLINE int comparColId(const void *key1, const void *key2) { +static FORCE_INLINE int32_t comparColId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) { @@ -116,26 +117,26 @@ static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) { // ----------------- SCHEMA BUILDER DEFINITION typedef struct { - int tCols; - int nCols; - int tlen; + int32_t tCols; + int32_t nCols; + int32_t tlen; uint16_t flen; uint16_t vlen; - int version; + int32_t version; STColumn *columns; } STSchemaBuilder; -#define TD_VTYPE_BITS 2 // val type +#define TD_VTYPE_BITS 2 // val type #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 -#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder +#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder #define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) #define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); +int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -143,17 +144,17 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); typedef uint64_t TKEY; -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID #define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_DELETE_FLAG (((TKEY)1) << 62) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) +#define TKEY_DELETE_FLAG (((TKEY)1) << 62) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) +#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) #define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) -#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) +#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) #define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) @@ -165,15 +166,15 @@ typedef uint64_t TKEY; // typedef uint64_t TKEY; #define TKEY TSKEY -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID #define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG)) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (false) +#define TKEY_IS_DELETED(tkey) (false) -#define tdGetTKEY(key) (key) +#define tdGetTKEY(key) (key) #define tdGetKey(tskey) (tskey) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) @@ -194,7 +195,7 @@ static FORCE_INLINE TKEY keyToTkey(TSKEY key) { return tdGetTKEY(lkey); } -static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { +static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) { TSKEY key1 = tdGetKey(*(TKEY *)tkey1); TSKEY key2 = tdGetKey(*(TKEY *)tkey2); @@ -225,17 +226,17 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 -#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) -#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) -#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) -#define dataRowKey(r) tdGetKey(dataRowTKey(r)) -#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) -#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) -#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) +#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 +#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) +#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) +#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) +#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) +#define dataRowKey(r) tdGetKey(dataRowTKey(r)) +#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) +#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) +#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) -#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) +#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); @@ -243,7 +244,7 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, +static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, int32_t offset) { assert(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; @@ -268,7 +269,7 @@ static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool } // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { +static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { return tdAppendDataColVal(row, value, true, type, offset); } @@ -281,25 +282,25 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o } } -static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) { return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset); } -static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; return tdGetRowDataOfCol(row, type, offset); } -static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; return isNull(tdGetRowDataOfCol(row, type, offset), type); } -static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; int16_t bytes = pSchema->columns[idx].bytes; @@ -307,8 +308,8 @@ static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSche setNull(tdGetRowDataOfCol(row, type, offset), type, bytes); } -static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src, - STSchema *pSrcSchema, int srcIdx) { +static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src, + STSchema *pSrcSchema, int32_t srcIdx) { int8_t type = pDstSchema->columns[dstIdx].type; assert(type == pSrcSchema->columns[srcIdx].type); void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); @@ -364,13 +365,13 @@ typedef struct SDataCol { uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM uint8_t reserve : 7; int16_t colId; // column ID - int bytes; // column data bytes defined - int offset; // data offset in a SDataRow (including the header size) - int spaceSize; // Total space size for this column - int len; // column data length + int32_t bytes; // column data bytes defined + int32_t offset; // data offset in a SDataRow (including the header size) + int32_t spaceSize; // Total space size for this column + int32_t len; // column data length VarDataOffsetT *dataOff; // For binary and nchar data, the offset in the data column - void * pData; // Actual data pointer - void * pBitmap; // Bitmap pointer + void *pData; // Actual data pointer + void *pBitmap; // Bitmap pointer TSKEY ts; // only used in last NULL column } SDataCol; @@ -378,17 +379,17 @@ typedef struct SDataCol { #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } -int tdAllocMemForCol(SDataCol *pCol, int maxPoints); +int32_t tdAllocMemForCol(SDataCol *pCol, int32_t maxPoints); -void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); -void *dataColSetOffset(SDataCol *pCol, int nEle); +void dataColInit(SDataCol *pDataCol, STColumn *pCol, int32_t maxPoints); +int32_t dataColAppendVal(SDataCol *pCol, const void *value, int32_t numOfRows, int32_t maxPoints); +void *dataColSetOffset(SDataCol *pCol, int32_t nEle); -bool isNEleNull(SDataCol *pCol, int nEle); +bool isNEleNull(SDataCol *pCol, int32_t nEle); #if 0 // Get the data pointer from a column-wised data -static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { +static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) { if (isAllRowsNull(pCol)) { return getNullValue(pCol->type); } @@ -399,7 +400,7 @@ static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { } } -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { +static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { assert(rows > 0); if (IS_VAR_DATA_TYPE(pDataCol->type)) { @@ -412,15 +413,15 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { typedef struct { col_id_t maxCols; // max number of columns col_id_t numOfCols; // Total number of cols - int maxPoints; // max number of points - int numOfRows; - int sversion; // TODO: set sversion + int32_t maxPoints; // max number of points + int32_t numOfRows; + int32_t sversion; // TODO: set sversion SDataCol *cols; } SDataCols; -#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data -#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) +#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { if (pCols->numOfRows) { return dataColsTKeyAt(pCols, 0); @@ -429,7 +430,7 @@ static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { } } -static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int row) { +static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int32_t row) { assert(row < pCols->numOfRows); return dataColsKeyAt(pCols, row); } @@ -458,12 +459,12 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { } } -SDataCols *tdNewDataCols(int maxCols, int maxRows); +SDataCols *tdNewDataCols(int32_t maxCols, int32_t maxRows); void tdResetDataCols(SDataCols *pCols); -int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| @@ -483,30 +484,30 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(TDRowLenT *)(r)) -#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) -#define kvRowSetLen(r, len) kvRowLen(r) = (len) -#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) -#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) -#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) +#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) +#define kvRowSetLen(r, len) kvRowLen(r) = (len) +#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) +#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) +#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) -#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) -#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) -#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) -#define kvRowKey(r) tdGetKey(kvRowTKey(r)) -#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) -#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) - -SKVRow tdKVRowDup(SKVRow row); -int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); -int tdEncodeKVRow(void **buf, SKVRow row); -void * tdDecodeKVRow(void *buf, SKVRow *row); -void tdSortKVRowByColIdx(SKVRow row); - -static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { +#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) +#define kvRowFree(r) tfree(r) +#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowKey(r) tdGetKey(kvRowTKey(r)) +#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) +#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) + +SKVRow tdKVRowDup(SKVRow row); +int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); +int32_t tdEncodeKVRow(void **buf, SKVRow row); +void *tdDecodeKVRow(void *buf, SKVRow *row); +void tdSortKVRowByColIdx(SKVRow row); + +static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { @@ -528,7 +529,7 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { #if 0 // offset here not include kvRow header length -static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, +static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, int32_t offset) { assert(value != NULL); int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; @@ -581,15 +582,15 @@ typedef struct { SColIdx *pColIdx; uint16_t alloc; uint16_t size; - void * buf; + void *buf; } SKVRowBuilder; -int tdInitKVRowBuilder(SKVRowBuilder *pBuilder); -void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); -void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); -SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); +int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); +void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); +void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); +SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); @@ -602,7 +603,7 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, pBuilder->nCols++; - int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; + int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; if (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; @@ -643,24 +644,24 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, typedef void *SMemRow; -#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) -#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) +#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) +#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) #define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) -#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) +#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) +#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) -#define SMEM_ROW_DATA 0x0U // SDataRow -#define SMEM_ROW_KV 0x01U // SKVRow +#define SMEM_ROW_DATA 0x0U // SDataRow +#define SMEM_ROW_KV 0x01U // SKVRow #define KVRatioConvert (0.9f) #define memRowType(r) ((*(uint8_t *)(r)) & 0x01) -#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory -#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) -#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) -#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) -#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) +#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory +#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) +#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) +#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) +#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag @@ -668,14 +669,14 @@ typedef void *SMemRow; POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow #define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 -#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 +#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 #define memRowDataTLen(r) \ ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen #define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) -#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) +#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen static FORCE_INLINE char *memRowEnd(SMemRow row) { @@ -686,14 +687,14 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { } } -#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) -#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) -#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version +#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) +#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) +#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version #define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) +#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) -#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) +#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) #define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) #define memRowSetTKey(r, k) \ do { \ @@ -704,11 +705,11 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { } \ } while (0) -#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) -#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) +#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) +#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) -#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) +#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) SMemRow tdMemRowDup(SMemRow row); void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); @@ -736,7 +737,7 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_ } } -static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, +static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, int8_t type, int32_t offset) { if (isDataRow(row)) { tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset); diff --git a/include/common/tep.h b/include/common/tep.h index 956cf6ec361531ed12bf20027f77d2136f8c8d7b..602df8ca53c7eab4e7881598286b27a934a675dc 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -1,14 +1,28 @@ -#ifndef TDENGINE_TEP_H -#define TDENGINE_TEP_H +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_EP_H_ +#define _TD_COMMON_EP_H_ + +#include "tcommon.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tmsg.h" -#include "common.h" - typedef struct SCorEpSet { int32_t version; SEpSet epSet; @@ -17,19 +31,16 @@ typedef struct SCorEpSet { typedef struct SBlockOrderInfo { int32_t order; int32_t colIndex; - SColumnInfoData *pColData; -// int32_t type; -// int32_t bytes; -// bool hasNull; + SColumnInfoData* pColData; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); -void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); +int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); +void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); -bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); +bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); -SEpSet getEpSet_s(SCorEpSet *pEpSet); +void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet); +SEpSet getEpSet_s(SCorEpSet* pEpSet); #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) @@ -41,7 +52,8 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { +static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, + SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { return false; } @@ -67,22 +79,27 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define colDataGet(p1_, r_) \ +#define BitmapLen(_n) (((_n) + ((1<> NBIT) + + +#define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); -int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); -void colDataTrim(SColumnInfoData* pColumnInfoData); +int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); +void colDataTrim(SColumnInfoData* pColumnInfoData); size_t colDataGetNumOfCols(const SSDataBlock* pBlock); size_t colDataGetNumOfRows(const SSDataBlock* pBlock); -int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize); SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); @@ -93,17 +110,20 @@ size_t blockDataGetRowSize(const SSDataBlock* pBlock); double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); -size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize); +SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); -void *blockDataDestroy(SSDataBlock *pBlock); +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); +SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); +void* blockDataDestroy(SSDataBlock* pBlock); #ifdef __cplusplus } #endif -#endif // TDENGINE_TEP_H +#endif /*_TD_COMMON_EP_H_*/ diff --git a/include/common/tglobal.h b/include/common/tglobal.h index a6cd04e006c4ccb423d5c95231ea3f10bf6a6518..e09f6d11bdea1f3dbf5a08ab37d3086c9c1d2ed4 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -16,14 +16,13 @@ #ifndef _TD_COMMON_GLOBAL_H_ #define _TD_COMMON_GLOBAL_H_ +#include "tarray.h" +#include "tdef.h" + #ifdef __cplusplus extern "C" { #endif -#include "tcfg.h" -#include "tdef.h" -#include "tarray.h" - // cluster extern char tsFirst[]; extern char tsSecond[]; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9ab78f3c9603011296c7ef450ba0d792dbc19043..a739e51d6b2688020e9a81cad47d487a3e6e00fc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -16,19 +16,19 @@ #ifndef _TD_COMMON_TAOS_MSG_H_ #define _TD_COMMON_TAOS_MSG_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include "encode.h" #include "taosdef.h" #include "taoserror.h" #include "tarray.h" #include "tcoding.h" +#include "tencode.h" #include "thash.h" #include "tlist.h" #include "trow.h" +#ifdef __cplusplus +extern "C" { +#endif + /* ------------------------ MESSAGE DEFINITIONS ------------------------ */ #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ @@ -48,8 +48,8 @@ extern "C" { #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -extern char* tMsgInfo[]; -extern int tMsgDict[]; +extern char* tMsgInfo[]; +extern int32_t tMsgDict[]; #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) @@ -655,7 +655,7 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp typedef struct { int32_t statusInterval; - int64_t checkTime; // 1970-01-01 00:00:00.000 + int64_t checkTime; // 1970-01-01 00:00:00.000 char timezone[TD_TIMEZONE_LEN]; // tsTimezone char locale[TD_LOCALE_LEN]; // tsLocale char charset[TD_LOCALE_LEN]; // tsCharset @@ -1117,13 +1117,13 @@ typedef struct { SArray* topicNames; // SArray } SCMSubscribeReq; -static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->consumerGroup); - for (int i = 0; i < pReq->topicNum; i++) { + for (int32_t i = 0; i < pReq->topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; @@ -1134,7 +1134,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeString(buf, &pReq->consumerGroup); pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); - for (int i = 0; i < pReq->topicNum; i++) { + for (int32_t i = 0; i < pReq->topicNum; i++) { char* name; buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); @@ -1153,10 +1153,10 @@ typedef struct { SMqSubTopic topics[]; } SCMSubscribeRsp; -static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pRsp->topicNum); - for (int i = 0; i < pRsp->topicNum; i++) { + for (int32_t i = 0; i < pRsp->topicNum; i++) { tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); @@ -1166,7 +1166,7 @@ static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribe static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { buf = taosDecodeFixedI32(buf, &pRsp->topicNum); - for (int i = 0; i < pRsp->topicNum; i++) { + for (int32_t i = 0; i < pRsp->topicNum; i++) { buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); @@ -1184,8 +1184,8 @@ typedef struct { char* physicalPlan; } SMVSubscribeReq; -static FORCE_INLINE int tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->topicId); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId); @@ -1397,8 +1397,8 @@ typedef struct SMqHbRsp { SEpSet epSet; } SMqHbRsp; -static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pRsp->status); tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); tlen += taosEncodeFixedI8(buf, pRsp->epChanged); @@ -1419,8 +1419,8 @@ typedef struct SMqHbOneTopicBatchRsp { SArray* rsps; // SArray } SMqHbOneTopicBatchRsp; -static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { + int32_t tlen = 0; tlen += taosEncodeString(buf, pBatchRsp->topicName); int32_t sz = taosArrayGetSize(pBatchRsp->rsps); tlen += taosEncodeFixedI32(buf, sz); @@ -1449,8 +1449,8 @@ typedef struct SMqHbBatchRsp { SArray* batchRsps; // SArray } SMqHbBatchRsp; -static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId); int32_t sz; tlen += taosEncodeFixedI32(buf, sz); @@ -1591,8 +1591,8 @@ typedef struct SMqHbVgInfo { int32_t vgId; } SMqHbVgInfo; -static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); return tlen; } @@ -1609,8 +1609,8 @@ typedef struct SMqHbTopicInfo { SArray* pVgInfo; } SMqHbTopicInfo; -static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); tlen += taosEncodeString(buf, pTopicInfo->name); @@ -1645,14 +1645,14 @@ typedef struct SMqHbMsg { SArray* pTopics; // SArray } SMqHbMsg; -static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pMsg->status); tlen += taosEncodeFixedI32(buf, pMsg->epoch); tlen += taosEncodeFixedI64(buf, pMsg->consumerId); int32_t sz = taosArrayGetSize(pMsg->pTopics); tlen += taosEncodeFixedI32(buf, sz); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); } @@ -1666,7 +1666,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { SMqHbTopicInfo topicInfo; buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); taosArrayPush(pMsg->pTopics, &topicInfo); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a3e44968f76fd43a161d7cd33f4ea5fc0304041b..cae186ba16ed7af71d888eee367f13b067b712d5 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -47,7 +47,7 @@ enum { #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) -int tMsgDict[] = { +int32_t tMsgDict[] = { #elif defined(TD_MSG_SEG_CODE_) @@ -127,7 +127,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STB_META, "mnode-stb-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) diff --git a/include/common/tname.h b/include/common/tname.h index 47028fbce18945f1b33f57d5d5ccaba2a730103a..6de38a68ee9ac782eb52e02f097d24633331ea13 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -13,26 +13,25 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TNAME_H -#define TDENGINE_TNAME_H +#ifndef _TD_COMMON_NAME_H_ +#define _TD_COMMON_NAME_H_ + +#include "tdef.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 -#include "tdef.h" -#include "tmsg.h" - -#define TSDB_DB_NAME_T 1 -#define TSDB_TABLE_NAME_T 2 - -#define T_NAME_ACCT 0x1u -#define T_NAME_DB 0x2u -#define T_NAME_TABLE 0x4u +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u typedef struct SName { - uint8_t type; //db_name_t, table_name_t + uint8_t type; // db_name_t, table_name_t int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; @@ -68,5 +67,4 @@ SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* nam } #endif - -#endif // TDENGINE_TNAME_H +#endif /*_TD_COMMON_NAME_H_*/ diff --git a/include/common/trequest.h b/include/common/trequest.h index 6932122bc92553255f6989746cbcb091b6f201c6..05abadcb4431f550f36b31c733f278f852e2221b 100644 --- a/include/common/trequest.h +++ b/include/common/trequest.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TREQUEST_H_ -#define _TD_TREQUEST_H_ +#ifndef _TD_COMMON_REQUEST_H_ +#define _TD_COMMON_REQUEST_H_ #ifdef __cplusplus extern "C" { @@ -36,7 +36,7 @@ void tdClearRBIter(SReqBatchIter *pIter); /* ------------------------ TYPES DEFINITION ------------------------ */ struct SReqBatchIter { - int iReq; + int32_t iReq; SReqBatch *pReqBatch; }; @@ -44,4 +44,4 @@ struct SReqBatchIter { } #endif -#endif /*_TD_TREQUEST_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_REQUEST_H_*/ \ No newline at end of file diff --git a/include/common/trow.h b/include/common/trow.h index fad74a359dad164cc18ad7bb1f06b90b116ace24..49bc2f3515a3304e8421d3f6f4cdc33a901be430 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -18,11 +18,11 @@ #include "os.h" #include "talgo.h" +#include "taosdef.h" #include "taoserror.h" #include "tbuffer.h" -#include "tdef.h" -#include "taosdef.h" #include "tdataformat.h" +#include "tdef.h" #include "tschema.h" #include "ttypes.h" #include "tutil.h" @@ -47,21 +47,21 @@ extern "C" { #define TD_VTYPE_NONE 0x0U // none or unknown/undefined #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NORM 0x02U // normal val: not none, not null -#define TD_VTYPE_MAX 0x03U // +#define TD_VTYPE_MAX 0x03U // #define TD_VTYPE_NONE_BYTE 0x0U #define TD_VTYPE_NULL_BYTE 0x55U #define TD_VTYPE_NORM_BYTE 0xAAU -#define TD_ROWS_ALL_NORM 0x01U +#define TD_ROWS_ALL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x0U -#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM +#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) -#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) -#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) +#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) +#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) -#define KvConvertRatio (0.9f) +#define KvConvertRatio (0.9f) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #ifdef TD_SUPPORT_BITMAP @@ -98,7 +98,7 @@ typedef void *SRow; typedef struct { TDRowValT valType; - void * val; + void *val; } SCellVal; typedef struct { @@ -157,41 +157,41 @@ typedef struct { int16_t nBitmaps; int16_t nBoundBitmaps; int32_t offset; - void * pBitmap; - void * pOffset; + void *pBitmap; + void *pOffset; int32_t extendedRowSize; } SRowBuilder; -#define TD_ROW_HEAD_LEN (sizeof(STSRow)) +#define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and -// (int)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. +// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) -#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) -#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) -#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) -#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) +#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) +#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) +#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) +#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n)) #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1) -#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) -#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) -#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) -#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) +#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) +#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) +#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) +#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) -#define TD_BOOL_STR(b) ((b) ? "true" : "false") +#define TD_BOOL_STR(b) ((b) ? "true" : "false") #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t)) @@ -217,7 +217,7 @@ static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType); static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType); -int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints); +int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -272,7 +272,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); @@ -310,7 +310,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -355,7 +355,6 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD * */ - /** * @brief * @@ -432,7 +431,6 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t pBuilder->rowType = TD_ROW_TP; } - } else { pBuilder->rowType = TD_ROW_TP; } @@ -618,7 +616,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa if (tdValIsNorm(valType, val, colType)) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN @@ -636,7 +634,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa // NULL/None value else { SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN const void *nullVal = getNullValue(colType); @@ -731,7 +729,7 @@ static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, v return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int compareKvRowColId(const void *key1, const void *key2) { +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { @@ -772,8 +770,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v typedef struct { STSchema *pSchema; - STSRow * pRow; - void * pBitmap; + STSRow *pRow; + void *pBitmap; uint32_t offset; col_id_t maxColId; col_id_t colIdx; // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1 @@ -793,8 +791,8 @@ static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; } -static int tdCompareColId(const void *arg1, const void *arg2) { - int colId = *(int *)arg1; +static int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; if (colId < pCol->colId) { @@ -878,7 +876,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy // internal static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { - STSRow * pRow = pIter->pRow; + STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow); @@ -959,11 +957,11 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); // Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int row) { +static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row) { if (isAllRowsNone(pCol)) { pVal->valType = TD_VTYPE_NULL; #ifdef TD_SUPPORT_READ2 - pVal->val = (void*)getNullValue(pCol->type); + pVal->val = (void *)getNullValue(pCol->type); #else pVal->val = NULL; #endif @@ -981,7 +979,7 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } else { pVal->valType = TD_VTYPE_NULL; #ifdef TD_SUPPORT_READ2 - pVal->val = (void*)getNullValue(pCol->type); + pVal->val = (void *)getNullValue(pCol->type); #else pVal->val = NULL; #endif @@ -1000,7 +998,8 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t return true; } -static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { +static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, + SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; @@ -1010,7 +1009,7 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o return true; } -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { +static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { ASSERT(rows > 0); int32_t result = 0; @@ -1068,7 +1067,7 @@ typedef struct { typedef struct { STSchema *pSchema; - STSRow * pRow; + STSRow *pRow; } STSRowReader; typedef struct { @@ -1079,15 +1078,15 @@ typedef struct { // STSRowBuilder #define trbInit(rt, allocator, endian, target, size) \ { .type = (rt), .bw = tbufInitWriter(allocator, endian), .pRow = (target) } -void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); -void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); -void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); -int trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); +void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); +void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); +void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); +int32_t trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); // STSRowReader #define tRowReaderInit(schema, row) \ { .schema = (schema), .row = (row) } -int tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); +int32_t tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); // STSRowBatchIter #define tRowBatchIterInit(pRB) \ diff --git a/include/common/tschema.h b/include/common/tschema.h index cfa0eeace2a0096492d4bb61e32359f0fe9e5e80..7a270fc9d9d247b5d2ea4483b36969225d5a880e 100644 --- a/include/common/tschema.h +++ b/include/common/tschema.h @@ -70,7 +70,7 @@ typedef struct { { .size = (capacity), .pSchema = (target) } void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver); void tSchemaBuilderSetSMA(bool sma, SArray *smaArray); -int tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment); +int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment); #endif diff --git a/include/common/ttime.h b/include/common/ttime.h index e22e1d6c4ed9aeb406abce8468d3f92f4599c298..a4b2aa6673c4413225ff0bfdd5b58851f400d2b0 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -16,13 +16,13 @@ #ifndef _TD_COMMON_TIME_H_ #define _TD_COMMON_TIME_H_ +#include "taosdef.h" +#include "tmsg.h" + #ifdef __cplusplus extern "C" { #endif -#include "taosdef.h" -#include "tmsg.h" - #define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') /* @@ -35,7 +35,7 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { return taosGetTimestampUs(); } else if (precision == TSDB_TIME_PRECISION_NANO) { return taosGetTimestampNs(); - }else { + } else { return taosGetTimestampMs(); } } @@ -56,4 +56,4 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec } #endif -#endif /*_TD_COMMON_TIME_H_*/ +#endif /*_TD_COMMON_TIME_H_*/ diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 35d8d00b509e15615b5098636d79c003fa77753a..fe5e1cc68cf571b4ac834229c76397fa3716b585 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTOKENDEF_H -#define TDENGINE_TTOKENDEF_H +#ifndef _TD_COMMON_TOKEN_H_ +#define _TD_COMMON_TOKEN_H_ #define TK_OR 1 #define TK_AND 2 @@ -135,6 +135,4 @@ #define TK_NIL 65535 -#endif - - +#endif /*_TD_COMMON_TOKEN_H_*/ diff --git a/include/common/ttszip.h b/include/common/ttszip.h index d83b0066d5b743155ea3739c38ef46add9194943..e9f430232054a4a13c8aec7eada9b452d4873ca8 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -13,19 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTSZIP_H -#define TDENGINE_TTSZIP_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_COMMON_TTSZIP_H_ +#define _TD_COMMON_TTSZIP_H_ #include "os.h" #include "tdef.h" #include "tvariant.h" -#define MEM_BUF_SIZE (1 << 20) -#define TS_COMP_FILE_MAGIC 0x87F5EC4C +#ifdef __cplusplus +extern "C" { +#endif + +#define MEM_BUF_SIZE (1 << 20) +#define TS_COMP_FILE_MAGIC 0x87F5EC4C #define TS_COMP_FILE_GROUP_MAX 512 typedef struct STSList { @@ -61,10 +61,10 @@ typedef struct STSBlock { * and the offset of int32_t type is enough */ typedef struct STSGroupBlockInfo { - int32_t id; // group id - int32_t offset; // offset set value in file - int32_t numOfBlocks; // number of total blocks - int32_t compLen; // compressed size + int32_t id; // group id + int32_t offset; // offset set value in file + int32_t numOfBlocks; // number of total blocks + int32_t compLen; // compressed size } STSGroupBlockInfo; typedef struct STSGroupBlockInfoEx { @@ -112,9 +112,9 @@ STSBuf* tsBufClone(STSBuf* pTSBuf); STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); -void tsBufFlush(STSBuf* pTSBuf); -void tsBufResetPos(STSBuf* pTSBuf); -bool tsBufNextPos(STSBuf* pTSBuf); +void tsBufFlush(STSBuf* pTSBuf); +void tsBufResetPos(STSBuf* pTSBuf); +bool tsBufNextPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag); @@ -144,4 +144,4 @@ bool tsBufIsValidElem(STSElem* pElem); } #endif -#endif // TDENGINE_TTSZIP_H +#endif /*_TD_COMMON_TTSZIP_H_*/ diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 39a5f0e2b8a42f96390b060929c9c4bcd10beeb6..5aa22bdcad699e7502d880786372d31a3545bd56 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -1,5 +1,20 @@ -#ifndef TDENGINE_TTYPE_H -#define TDENGINE_TTYPE_H +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_TTYPE_H_ +#define _TD_COMMON_TTYPE_H_ #ifdef __cplusplus extern "C" { @@ -15,11 +30,6 @@ typedef uint8_t TDRowValT; typedef uint16_t col_id_t; typedef int8_t col_type_t; -typedef struct tstr { - VarDataLenT len; - char data[]; -} tstr; - #pragma pack(push, 1) typedef struct { VarDataLenT len; @@ -32,19 +42,18 @@ typedef struct { } SNCharNullT; #pragma pack(pop) -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) -#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) -#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) - -#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) -#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) +#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) // this data type is internally used only in 'in' query to hold the values -#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) -#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) +#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) +#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ do { \ @@ -62,7 +71,7 @@ typedef struct { case TSDB_DATA_TYPE_USMALLINT: \ (_v) = (_finalType)GET_UINT16_VAL(_data); \ break; \ - case TSDB_DATA_TYPE_TIMESTAMP:\ + case TSDB_DATA_TYPE_TIMESTAMP: \ case TSDB_DATA_TYPE_BIGINT: \ (_v) = (_finalType)(GET_INT64_VAL(_data)); \ break; \ @@ -91,7 +100,7 @@ typedef struct { do { \ switch (_type) { \ case TSDB_DATA_TYPE_BOOL: \ - *(bool *)(_v) = (bool)(_data); \ + *(bool *)(_v) = (bool)(_data); \ break; \ case TSDB_DATA_TYPE_TINYINT: \ *(int8_t *)(_v) = (int8_t)(_data); \ @@ -135,18 +144,19 @@ typedef struct { #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) -#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) -#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) -#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) -#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) -#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) -#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) -#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) -#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) - -#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) +#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) + +#define IS_CONVERT_AS_SIGNED(_t) \ + (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) static FORCE_INLINE bool isNull(const void *val, int32_t type) { @@ -167,17 +177,17 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) { case TSDB_DATA_TYPE_DOUBLE: return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; case TSDB_DATA_TYPE_NCHAR: - return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + return varDataLen(val) == sizeof(int32_t) && *(uint32_t *)varDataVal(val) == TSDB_DATA_NCHAR_NULL; case TSDB_DATA_TYPE_BINARY: - return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *)varDataVal(val) == TSDB_DATA_BINARY_NULL; case TSDB_DATA_TYPE_UTINYINT: - return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + return *(uint8_t *)val == TSDB_DATA_UTINYINT_NULL; case TSDB_DATA_TYPE_USMALLINT: - return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + return *(uint16_t *)val == TSDB_DATA_USMALLINT_NULL; case TSDB_DATA_TYPE_UINT: - return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + return *(uint32_t *)val == TSDB_DATA_UINT_NULL; case TSDB_DATA_TYPE_UBIGINT: - return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + return *(uint64_t *)val == TSDB_DATA_UBIGINT_NULL; default: return false; @@ -188,32 +198,31 @@ typedef struct tDataTypeDescriptor { int16_t type; int16_t nameLen; int32_t bytes; - char * name; + char *name; int64_t minValue; int64_t maxValue; - int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize); - int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize); - void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minindex, int16_t *maxindex, int16_t *numofnull); + int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, + int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[15]; bool isValidDataType(int32_t type); -void setVardataNull(void* val, int32_t type); -void setNull(void *val, int32_t type, int32_t bytes); -void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); +void setVardataNull(void *val, int32_t type); +void setNull(void *val, int32_t type, int32_t bytes); +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); const void *getNullValue(int32_t type); -void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); -void* getDataMin(int32_t type); -void* getDataMax(int32_t type); - +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf); +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); +void *getDataMin(int32_t type); +void *getDataMax(int32_t type); #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) #define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL) @@ -222,4 +231,4 @@ void* getDataMax(int32_t type); } #endif -#endif // TDENGINE_TTYPE_H +#endif /*_TD_COMMON_TTYPE_H_*/ diff --git a/include/common/tvariant.h b/include/common/tvariant.h index e068c6eb6ca4cc56b3eae27d5ca213517ad86752..995015fe6301b950f2ae415aef791296e6621d28 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -13,34 +13,34 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TVARIANT_H -#define TDENGINE_TVARIANT_H +#ifndef _TD_COMMON_VARIANT_H_ +#define _TD_COMMON_VARIANT_H_ + +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#include "tarray.h" - // variant, each number/string/field_id has a corresponding struct during parsing sql typedef struct SVariant { - uint32_t nType; - int32_t nLen; // only used for string, for number, it is useless + uint32_t nType; + int32_t nLen; // only used for string, for number, it is useless union { int64_t i; uint64_t u; double d; char *pz; wchar_t *wpz; - SArray *arr; // only for 'in' query to hold value list, not value for a field + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } SVariant; -int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* issigned); +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *issigned); bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type); +void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); @@ -48,7 +48,7 @@ void taosVariantDestroy(SVariant *pV); void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); -int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2); +int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2); int32_t taosVariantToString(SVariant *pVar, char *dst); @@ -64,4 +64,4 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type); } #endif -#endif // TDENGINE_TVARIANT_H +#endif /*_TD_COMMON_VARIANT_H_*/ diff --git a/include/common/type/type.h b/include/common/type/type.h deleted file mode 100644 index 3cbea6edbb41252a303615a4db3c45effed7e2bd..0000000000000000000000000000000000000000 --- a/include/common/type/type.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TYPE_H_ -#define _TD_TYPE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TYPE_H_*/ \ No newline at end of file diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index bb0f66ca09cd550c204e811bb936860ad89829f5..068e9a69170bfd1659b2961bef220b765d74c35e 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -17,7 +17,6 @@ #define _TD_DNODE_H_ #include "tdef.h" -#include "tcfg.h" #ifdef __cplusplus extern "C" { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b6054c02b4ae54927c5f0e7ff1d41adc33243e6b..ebe7c92d31b26779f4f478c332acafc862ba71a7 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -24,7 +24,7 @@ extern "C" { #include "taosdef.h" #include "query.h" #include "tname.h" -#include "common.h" +#include "tcommon.h" #include "tarray.h" #include "thash.h" #include "tmsg.h" diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d9e1957e5ddbb21d8e49786ea11901c06f3087ee..a4c0be27abeae8566aa5efabfb20972107e70b2c 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -31,6 +31,7 @@ typedef struct SReadHandle { void* reader; void* meta; } SReadHandle; + /** * Create the exec task for streaming mode * @param pMsg @@ -40,12 +41,22 @@ typedef struct SReadHandle { qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); /** - * + * Set the input data block for the stream scan. * @param tinfo * @param input * @return */ -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); + +/** + * Update the table id list, add or remove. + * + * @param tinfo + * @param id + * @param isAdd + * @return + */ +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e970a0d693db322a4cfa8a38e5aa28f778ab4197..62063ef99a67b961b852a3870600d4d4b850ed7b 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "tvariant.h" #include "tbuffer.h" @@ -138,8 +138,10 @@ extern SFunctionFpSet fpSet[1]; // sql function runtime context typedef struct SqlFunctionCtx { + int32_t startRow; int32_t size; // number of rows - void * pInput; // input data buffer + SColumnInfoData* pInput; + uint32_t order; // asc|desc int16_t inputType; int16_t inputBytes; @@ -227,11 +229,16 @@ typedef struct SAggFunctionInfo { } SAggFunctionInfo; typedef struct SScalarParam { - void* data; - bool colData; - int32_t num; - int32_t type; - int32_t bytes; + void *data; + union { + SColumnInfoData *columnData; + void *data; + } orig; + char *bitmap; + bool dataInBlock; + int32_t num; + int32_t type; + int32_t bytes; } SScalarParam; typedef struct SScalarFunctionInfo { diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 1902c11cc68097b63141d640ce7a8bdf4656d713..e55c0b9e7667e934236c771d8980ace53aa57ca9 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -25,9 +25,17 @@ extern "C" { typedef struct SFilterInfo SFilterInfo; - +/* +pNode will be freed in API; +*pRes need to freed in caller +*/ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); + +/* +pDst need to freed in caller +*/ +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); + int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 289cfd9b04bfaf34fc4e8bb1ad5e07cc00ba4959..b6d10c81bffb230c5e8408d4773cd8ceccf25baa 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -16,7 +16,7 @@ #ifndef _TD_TFS_H_ #define _TD_TFS_H_ -#include "tcfg.h" +#include "tdef.h" #ifdef __cplusplus extern "C" { diff --git a/include/os/os.h b/include/os/os.h index d96f7efb51dd0ca36d307308ba005d4fd7f628b6..f020af5a657dcb3bcaf997ba0a68e6e57ca31b23 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_H -#define TDENGINE_OS_H +#ifndef _TD_OS_H_ +#define _TD_OS_H_ #ifdef __cplusplus extern "C" { @@ -82,4 +82,4 @@ void osInit(); } #endif -#endif +#endif /*_TD_OS_H_*/ diff --git a/include/os/osFile.h b/include/os/osFile.h index cedf26d04aa1573c3ee0a066885c0a995fd5597c..cbd5d693c5aea2d8faa000b006a6a08575932391 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -43,6 +43,7 @@ typedef struct TdFile *TdFilePtr; #define TD_FILE_TEXT 0x0020 #define TD_FILE_AUTO_DEL 0x0040 #define TD_FILE_EXCL 0x0080 +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile int32_t taosLockFile(TdFilePtr pFile); int32_t taosUnLockFile(TdFilePtr pFile); diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 5f1d5a9a8a6a7028295cac28d6711f6171be7307..6100419035ff9901bf6a6d5c3871496d0edcd533 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -20,12 +20,12 @@ extern "C" { #endif -#define tfree(x) \ - do { \ - if (x) { \ +#define tfree(x) \ + do { \ + if (x) { \ free((void *)(x)); \ (x) = 0; \ - } \ + } \ } while (0) #ifdef __cplusplus diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 78112fc7a0f58d0659cdbd0043c47e814ca69ca0..7fb9a2202d38760e5f1c760fe9d7f8eaa4096f46 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -16,12 +16,12 @@ #ifndef _TD_OS_SEMPHONE_H_ #define _TD_OS_SEMPHONE_H_ +#include + #ifdef __cplusplus extern "C" { #endif -#include - #if defined (_TD_DARWIN_64) typedef struct tsem_s *tsem_t; int tsem_init(tsem_t *sem, int pshared, unsigned int value); diff --git a/include/os/osSocket.h b/include/os/osSocket.h index af99e4b47419350e408d8809e8d593d0b2a3996a..395874a88c5bded90fad93e781ba187e7ec911b1 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -16,10 +16,6 @@ #ifndef _TD_OS_SOCKET_H_ #define _TD_OS_SOCKET_H_ -#ifdef __cplusplus -extern "C" { -#endif - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include "winsock2.h" #include @@ -30,6 +26,10 @@ extern "C" { #include #endif +#ifdef __cplusplus +extern "C" { +#endif + #define TAOS_EPOLL_WAIT_TIME 500 typedef int32_t SOCKET; typedef SOCKET EpollFd; diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9f9061d2437f64bbad065ca7a28e35e3504dcec8..a0771dc7340562787a0e8bd69fbb880998fb8dcc 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -16,12 +16,12 @@ #ifndef _TD_OS_SYSINFO_H_ #define _TD_OS_SYSINFO_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" - typedef struct { int64_t total; int64_t used; diff --git a/include/os/osThread.h b/include/os/osThread.h index 79834dc9a59f00c0aefa452c2427ea45ac35c478..cccc13755d3a33f0ecaa1173df391228d3c90da9 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -16,12 +16,12 @@ #ifndef _TD_OS_THREAD_H_ #define _TD_OS_THREAD_H_ +#include + #ifdef __cplusplus extern "C" { #endif -#include - #ifdef __cplusplus } #endif diff --git a/include/util/exception.h b/include/util/exception.h deleted file mode 100644 index 8012e3431591099a645373092fb34ac0fb66ea77..0000000000000000000000000000000000000000 --- a/include/util/exception.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_EXCEPTION_H -#define _TD_UTIL_EXCEPTION_H - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * cleanup actions - */ -typedef struct SCleanupAction { - bool failOnly; - uint8_t wrapper; - uint16_t reserved; - void* func; - union { - void* Ptr; - bool Bool; - char Char; - int8_t Int8; - uint8_t Uint8; - int16_t Int16; - uint16_t Uint16; - int Int; - unsigned int Uint; - int32_t Int32; - uint32_t Uint32; - int64_t Int64; - uint64_t Uint64; - float Float; - double Double; - } arg1, arg2; -} SCleanupAction; - - -/* - * exception hander registration - */ -typedef struct SExceptionNode { - struct SExceptionNode* prev; - jmp_buf jb; - int32_t code; - int32_t maxCleanupAction; - int32_t numCleanupAction; - SCleanupAction* cleanupActions; -} SExceptionNode; - -//////////////////////////////////////////////////////////////////////////////// -// functions & macros for auto-cleanup - -void cleanupPush_void_ptr_ptr ( bool failOnly, void* func, void* arg1, void* arg2 ); -void cleanupPush_void_ptr_bool ( bool failOnly, void* func, void* arg1, bool arg2 ); -void cleanupPush_void_ptr ( bool failOnly, void* func, void* arg ); -void cleanupPush_int_int ( bool failOnly, void* func, int arg ); -void cleanupPush_void ( bool failOnly, void* func ); -void cleanupPush_int_ptr ( bool failOnly, void* func, void* arg ); - -int32_t cleanupGetActionCount(); -void cleanupExecuteTo( int32_t anchor, bool failed ); -void cleanupExecute( SExceptionNode* node, bool failed ); -bool cleanupExceedLimit(); - -#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) ) -#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) ) -#define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) ) -#define CLEANUP_PUSH_INT_INT( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) ) -#define CLEANUP_PUSH_VOID( failOnly, func ) cleanupPush_void( (failOnly), (void*)(func) ) -#define CLEANUP_PUSH_INT_PTR( failOnly, func, arg ) cleanupPush_int_ptr( (failOnly), (void*)(func), (void*)(arg) ) -#define CLEANUP_PUSH_FREE( failOnly, arg ) cleanupPush_void_ptr( (failOnly), free, (void*)(arg) ) -#define CLEANUP_PUSH_CLOSE( failOnly, arg ) cleanupPush_int_int( (failOnly), close, (int)(arg) ) -#define CLEANUP_PUSH_FCLOSE( failOnly, arg ) cleanupPush_int_ptr( (failOnly), fclose, (void*)(arg) ) - -#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() -#define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) ) -#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit() - -//////////////////////////////////////////////////////////////////////////////// -// functions & macros for exception handling - -void exceptionPushNode( SExceptionNode* node ); -int32_t exceptionPopNode(); -void exceptionThrow( int32_t code ); - -#define TRY(maxCleanupActions) do { \ - SExceptionNode exceptionNode = { 0 }; \ - SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \ - exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \ - exceptionNode.cleanupActions = cleanupActions; \ - exceptionPushNode( &exceptionNode ); \ - int caughtException = setjmp( exceptionNode.jb ); \ - if( caughtException == 0 ) - -#define CATCH( code ) int32_t code = exceptionPopNode(); \ - if( caughtException == 1 ) - -#define FINALLY( code ) int32_t code = exceptionPopNode(); - -#define END_TRY } while( 0 ); - -#define THROW( x ) exceptionThrow( (x) ) -#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1)) -#define CLEANUP_EXECUTE() cleanupExecute( &exceptionNode, CAUGHT_EXCEPTION() ) - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_EXCEPTION_H*/ diff --git a/include/util/talgo.h b/include/util/talgo.h index 575f98b486f90b2eb4a830d0bb39868fcdc0c137..0fd44a6e91d664580a23188061cf14b9637919ab 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -13,16 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_TALGO_H -#define _TD_UTIL_TALGO_H +#ifndef _TD_UTIL_TALGO_H_ +#define _TD_UTIL_TALGO_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif #ifndef __COMPAR_FN_T -# define __COMPAR_FN_T -typedef int (*__compar_fn_t) (const void *, const void *); +#define __COMPAR_FN_T +typedef int32_t (*__compar_fn_t)(const void *, const void *); #endif #define TD_EQ 0x1 @@ -45,7 +47,7 @@ typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param); * @param param * @param comparFn */ -void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn); +void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); /** * binary search, with range support @@ -58,7 +60,7 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex * @param flags * @return */ -void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags); +void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags); /** * adjust heap @@ -74,7 +76,8 @@ void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, * @param maxroot: if heap is max root heap * @return */ -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -89,10 +92,11 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @param maxroot: if heap is max root heap * @return */ -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); - +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + const void *parswap, __ext_swap_fn_t swap, bool maxroot); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_TALGO_H*/ + +#endif /*_TD_UTIL_TALGO_H_*/ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7b42f3fea0fc46beeeae63aa6e4f53f32de89926..5fe69284a235b2ff19b5a0c604da80db68152b96 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -235,6 +235,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +// mnode-infoSchema +#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) + // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C1) @@ -265,6 +268,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) + + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) diff --git a/include/util/tarray.h b/include/util/tarray.h index 6d6120a49b5c7be84cf97a2af0cc7f44e6298966..521e54040d6abd8ac53af11d830325bc3b5c053c 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -13,16 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_ARRAY_H -#define _TD_UTIL_ARRAY_H +#ifndef _TD_UTIL_ARRAY_H_ +#define _TD_UTIL_ARRAY_H_ + +#include "talgo.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "talgo.h" - #if 0 #define TARRAY(TYPE) \ struct { \ @@ -31,20 +30,20 @@ extern "C" { struct TYPE* td_array_data_; \ } -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ +#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ +#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ #define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) #endif -#define TARRAY_MIN_SIZE 8 +#define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) -#define TARRAY_GET_START(array) ((array)->pData) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; - size_t capacity; - size_t elemSize; + uint32_t capacity; + uint32_t elemSize; void* pData; } SArray; @@ -70,7 +69,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); * @param nEles * @return */ -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles); +void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles); /** * @@ -238,7 +237,7 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); * @param compar * @param key */ -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags); /** * search the array, return index of the element @@ -246,14 +245,14 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa * @param compar * @param key */ -int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags); /** * search the array * @param pArray * @param key */ -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags); /** * sort the pointer data in the array @@ -269,4 +268,4 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par } #endif -#endif /*_TD_UTIL_ARRAY_H*/ +#endif /*_TD_UTIL_ARRAY_H_*/ diff --git a/source/util/inc/utilInt.h b/include/util/tbase64.h similarity index 74% rename from source/util/inc/utilInt.h rename to include/util/tbase64.h index ae2bad792d8777283b9289516cb748ce21142651..22b8f95c5a8868ccd508841df091674d0dab6a76 100644 --- a/source/util/inc/utilInt.h +++ b/include/util/tbase64.h @@ -13,15 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_INT_H_ -#define _TD_UTIL_INT_H_ +#ifndef _TD_UTIL_BASE64_H_ +#define _TD_UTIL_BASE64_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif +uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen); +char *base64_encode(const uint8_t *value, int32_t vlen); + #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_INT_H_*/ \ No newline at end of file +#endif /*_TD_UTIL_BASE64_H_*/ \ No newline at end of file diff --git a/include/util/tbuffer.h b/include/util/tbuffer.h index 6bb7f67e3d52549bac2a381bc4d65ad1b6263bf5..2ed1b7326bdb27ffe182b30d67a9369680898d23 100644 --- a/include/util/tbuffer.h +++ b/include/util/tbuffer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_BUFFER_H -#define _TD_UTIL_BUFFER_H +#ifndef _TD_UTIL_BUFFER_H_ +#define _TD_UTIL_BUFFER_H_ #include "os.h" @@ -26,9 +26,9 @@ extern "C" { // usage example /* #include -#include "exception.h" +#include "texception.h" -int main( int argc, char** argv ) { +int32_t main( int32_t argc, char** argv ) { SBufferWriter bw = tbufInitWriter( NULL, false ); TRY( 1 ) { @@ -39,7 +39,7 @@ int main( int argc, char** argv ) { // reserve space for the interger count size_t pos = tbufReserve( &bw, sizeof(int32_t) ); // write 5 integers to the buffer - for( int i = 0; i < 5; i++) { + for( int32_t i = 0; i < 5; i++) { tbufWriteInt32( &bw, i ); } // write the integer count to buffer at reserved position @@ -55,7 +55,7 @@ int main( int argc, char** argv ) { SBufferReader br = tbufInitReader( data, size, false ); // read & print out all integers int32_t count = tbufReadInt32( &br ); - for( int i = 0; i < count; i++ ) { + for( int32_t i = 0; i < count; i++ ) { printf( "%d\n", tbufReadInt32(&br) ); } // read & print out a string @@ -87,12 +87,10 @@ typedef struct SBufferWriter { void* (*allocator)(void*, size_t); } SBufferWriter; -//////////////////////////////////////////////////////////////////////////////// // common functions & macros for both reader & writer #define tbufTell(buf) ((buf)->pos) -//////////////////////////////////////////////////////////////////////////////// /* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */ // *Allocator*, function to allocate memory, will use 'realloc' if NULL // *Endian*, if true, writer functions of primitive types will do 'hton' automatically @@ -167,4 +165,4 @@ double tbufReadDouble(SBufferReader* buf); } #endif -#endif /*_TD_UTIL_BUFFER_H*/ +#endif /*_TD_UTIL_BUFFER_H_*/ diff --git a/include/util/tcache.h b/include/util/tcache.h index 11f9044d2838e8f73e9fcb01f6dc554b1de6aeff..0de3ab3a2801754f35f894ab3e3a02275cc5020b 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CACHE_H -#define _TD_UTIL_CACHE_H +#ifndef _TD_UTIL_CACHE_H_ +#define _TD_UTIL_CACHE_H_ + +#include "thash.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tlockfree.h" -#include "thash.h" - -#if defined(_TD_ARM_32) - #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT - #define TSDB_CACHE_PTR_TYPE int32_t +#if defined(_TD_ARM_32) +#define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT +#define TSDB_CACHE_PTR_TYPE int32_t #else - #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT - #define TSDB_CACHE_PTR_TYPE int64_t +#define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT +#define TSDB_CACHE_PTR_TYPE int64_t #endif -typedef void (*__cache_free_fn_t)(void*); -typedef void (*__cache_trav_fn_t)(void*, void*); +typedef void (*__cache_free_fn_t)(void *); +typedef void (*__cache_trav_fn_t)(void *, void *); typedef struct SCacheStatis { int64_t missCount; @@ -45,17 +43,17 @@ typedef struct SCacheStatis { struct STrashElem; typedef struct SCacheDataNode { - uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t lifespan; // life duration when this element should be remove from cache - uint64_t expireTime; // expire time + uint64_t addedTime; // the added time when this element is added or updated into cache + uint64_t lifespan; // life duration when this element should be remove from cache + uint64_t expireTime; // expire time uint64_t signature; - struct STrashElem *pTNodeHeader; // point to trash node head - uint16_t keySize: 15; // max key size: 32kb - bool inTrashcan: 1;// denote if it is in trash or not - uint32_t size; // allocated size for current SCacheDataNode + struct STrashElem *pTNodeHeader; // point to trash node head + uint16_t keySize : 15; // max key size: 32kb + bool inTrashcan : 1; // denote if it is in trash or not + uint32_t size; // allocated size for current SCacheDataNode T_REF_DECLARE() - char *key; - char data[]; + char *key; + char data[]; } SCacheDataNode; typedef struct STrashElem { @@ -73,22 +71,22 @@ typedef struct STrashElem { * when the node in pTrash does not be referenced, it will be release at the expired expiredTime */ typedef struct { - int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. - int64_t refreshTime; - STrashElem * pTrash; - char* name; - SCacheStatis statistics; - SHashObj * pHashTable; + int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. + int64_t refreshTime; + STrashElem *pTrash; + char *name; + SCacheStatis statistics; + SHashObj *pHashTable; __cache_free_fn_t freeFp; - uint32_t numOfElemsInTrash; // number of element in trash - uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; - bool extendLifespan; // auto extend life span when one item is accessed. - int64_t checkTick; // tick used to record the check times of the refresh threads + uint32_t numOfElemsInTrash; // number of element in trash + uint8_t deleting; // set the deleting flag to stop refreshing ASAP. + pthread_t refreshWorker; + bool extendLifespan; // auto extend life span when one item is accessed. + int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) pthread_rwlock_t lock; #else - pthread_mutex_t lock; + pthread_mutex_t lock; #endif } SCacheObj; @@ -101,7 +99,8 @@ typedef struct { * @param fn free resource callback function * @return */ -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, + const char *cacheName); /** * add data into cache @@ -113,7 +112,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, + int32_t durationMS); /** * get data from cache @@ -177,7 +177,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj); * @param fp * @return */ -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1); +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); /** * stop background refresh worker thread @@ -188,4 +188,4 @@ void taosStopCacheRefreshWorker(); } #endif -#endif /*_TD_UTIL_CACHE_H*/ +#endif /*_TD_UTIL_CACHE_H_*/ diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index fb6c85edcda5e70de8ba9ad18e8dfd1b9317e4ed..28fb784c467bd24fe92be2719cb6f983f08e0181 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -13,24 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CHECKSUM_H -#define _TD_UTIL_CHECKSUM_H +#ifndef _TD_UTIL_CHECKSUM_H_ +#define _TD_UTIL_CHECKSUM_H_ + +#include "tcrc32c.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tcrc32c.h" -#include "tutil.h" - typedef uint32_t TSCKSUM; static FORCE_INLINE TSCKSUM taosCalcChecksum(TSCKSUM csi, const uint8_t *stream, uint32_t ssize) { return (*crc32c)(csi, stream, (size_t)ssize); } -static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uint32_t ssize) { +static FORCE_INLINE int32_t taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uint32_t ssize) { if (ssize < sizeof(TSCKSUM)) return -1; *((TSCKSUM *)(stream + ssize - sizeof(TSCKSUM))) = (*crc32c)(csi, stream, (size_t)(ssize - sizeof(TSCKSUM))); @@ -38,11 +36,11 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin return 0; } -static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { +static FORCE_INLINE int32_t taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { return (checksum != (*crc32c)(0, stream, (size_t)ssize)); } -static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { +static FORCE_INLINE int32_t taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { if (ssize < sizeof(TSCKSUM)) return 0; #if (_WIN64) @@ -56,4 +54,4 @@ static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t s } #endif -#endif /*_TD_UTIL_CHECKSUM_H*/ +#endif /*_TD_UTIL_CHECKSUM_H_*/ diff --git a/include/util/tcoding.h b/include/util/tcoding.h index fed9d12cee62a01a2b92215326f4465beb760a1a..943a1f9ecad450f33b4f0e0eb9ef28fae60bd813 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -12,29 +12,30 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_CODING_H -#define _TD_UTIL_CODING_H + +#ifndef _TD_UTIL_CODING_H_ +#define _TD_UTIL_CODING_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -#define ENCODE_LIMIT (((uint8_t)1) << 7) +#define ENCODE_LIMIT (((uint8_t)1) << 7) #define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode /* ------------------------ LEGACY CODES ------------------------ */ #if 1 // ---- Fixed U8 -static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU8(void **buf, uint8_t value) { if (buf != NULL) { ((uint8_t *)(*buf))[0] = value; *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { @@ -43,12 +44,12 @@ static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { } // ---- Fixed I8 -static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI8(void **buf, int8_t value) { if (buf != NULL) { ((int8_t *)(*buf))[0] = value; *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { @@ -57,7 +58,7 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { } // ---- Fixed U16 -static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -68,7 +69,7 @@ static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { @@ -83,7 +84,7 @@ static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { } // ---- Fixed I16 -static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } @@ -95,7 +96,7 @@ static FORCE_INLINE void *taosDecodeFixedI16(const void *buf, int16_t *value) { } // ---- Fixed U32 -static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU32(void **buf, uint32_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -108,7 +109,7 @@ static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { @@ -125,7 +126,7 @@ static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { } // ---- Fixed I32 -static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } @@ -137,7 +138,7 @@ static FORCE_INLINE void *taosDecodeFixedI32(const void *buf, int32_t *value) { } // ---- Fixed U64 -static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU64(void **buf, uint64_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -155,7 +156,7 @@ static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { @@ -176,7 +177,7 @@ static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { } // ---- Fixed I64 -static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } @@ -188,8 +189,8 @@ static FORCE_INLINE void *taosDecodeFixedI64(const void *buf, int64_t *value) { } // ---- Variant U16 -static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU16(void **buf, uint16_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; @@ -206,7 +207,7 @@ static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { } static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) { - int i = 0; + int32_t i = 0; uint16_t tval = 0; *value = 0; while (i < 3) { @@ -224,7 +225,7 @@ static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) } // ---- Variant I16 -static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } @@ -236,8 +237,8 @@ static FORCE_INLINE void *taosDecodeVariantI16(const void *buf, int16_t *value) } // ---- Variant U32 -static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU32(void **buf, uint32_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; @@ -254,7 +255,7 @@ static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { } static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) { - int i = 0; + int32_t i = 0; uint32_t tval = 0; *value = 0; while (i < 5) { @@ -272,7 +273,7 @@ static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) } // ---- Variant I32 -static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } @@ -284,8 +285,8 @@ static FORCE_INLINE void *taosDecodeVariantI32(const void *buf, int32_t *value) } // ---- Variant U64 -static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU64(void **buf, uint64_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; @@ -302,7 +303,7 @@ static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { } static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) { - int i = 0; + int32_t i = 0; uint64_t tval = 0; *value = 0; while (i < 10) { @@ -320,7 +321,7 @@ static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) } // ---- Variant I64 -static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } @@ -332,16 +333,16 @@ static FORCE_INLINE void *taosDecodeVariantI64(const void *buf, int64_t *value) } // ---- string -static FORCE_INLINE int taosEncodeString(void **buf, const char *value) { - int tlen = 0; - size_t size = strlen(value); +static FORCE_INLINE int32_t taosEncodeString(void **buf, const char *value) { + int32_t tlen = 0; + size_t size = strlen(value); tlen += taosEncodeVariantU64(buf, size); if (buf != NULL) { memcpy(*buf, value, size); *buf = POINTER_SHIFT(*buf, size); } - tlen += (int)size; + tlen += (int32_t)size; return tlen; } @@ -372,14 +373,14 @@ static FORCE_INLINE void *taosDecodeStringTo(const void *buf, char *value) { } // ---- binary -static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { + int32_t tlen = 0; if (buf != NULL) { memcpy(*buf, value, valueLen); *buf = POINTER_SHIFT(*buf, valueLen); } - tlen += (int)valueLen; + tlen += (int32_t)valueLen; return tlen; } @@ -403,4 +404,4 @@ static FORCE_INLINE void *taosDecodeBinaryTo(const void *buf, void *value, int32 } #endif -#endif /*_TD_UTIL_CODING_H*/ +#endif /*_TD_UTIL_CODING_H_*/ diff --git a/include/util/compare.h b/include/util/tcompare.h similarity index 90% rename from include/util/compare.h rename to include/util/tcompare.h index ece8ffaf426193495cfd5318f9c3d76ba9364eb1..4c80eeb4f601034852acebd52b2e18b74c52d424 100644 --- a/include/util/compare.h +++ b/include/util/tcompare.h @@ -17,12 +17,12 @@ #define _TD_UTIL_COMPARE_H_ #include "os.h" +#include "taos.h" #ifdef __cplusplus extern "C" { #endif - #define TSDB_PATTERN_MATCH 0 #define TSDB_PATTERN_NOMATCH 1 #define TSDB_PATTERN_NOWILDCARDMATCH 2 @@ -99,7 +99,15 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); +int32_t compareStrPatternMatch(const void *pLeft, const void *pRight); +int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); + +int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); +int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); + __compar_fn_t getComparFunc(int32_t type, int32_t optr); +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); +int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); #ifdef __cplusplus } diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 4cb11935783504ba2792394baaf3942a560a7279..d7ba4dfa3fd9b255ded7cfe58cd2ad859575d445 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -13,22 +13,23 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_COMPRESSION_H -#define _TD_UTIL_COMPRESSION_H +#ifndef _TD_UTIL_COMPRESSION_H_ +#define _TD_UTIL_COMPRESSION_H_ + +#include "os.h" +#include "taos.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { #endif -#include "taos.h" -#include "tutil.h" - #define COMP_OVERFLOW_BYTES 2 -#define BITS_PER_BYTE 8 +#define BITS_PER_BYTE 8 // Masks #define INT64MASK(_x) ((((uint64_t)1) << _x) - 1) #define INT32MASK(_x) (((uint32_t)1 << _x) - 1) -#define INT8MASK(_x) (((uint8_t)1 << _x) - 1) +#define INT8MASK(_x) (((uint8_t)1 << _x) - 1) // Compression algorithm #define NO_COMPRESSION 0 #define ONE_STAGE_COMP 1 @@ -41,48 +42,51 @@ extern "C" { // // compression data mode save first byte lower 1 bit -#define MODE_NOCOMPRESS 0 // original data -#define MODE_COMPRESS 1 // compatible old compress +#define MODE_NOCOMPRESS 0 // original data +#define MODE_COMPRESS 1 // compatible old compress // compression algorithm save first byte higher 7 bit -#define ALGO_SZ_LOSSY 1 // SZ compress - -#define HEAD_MODE(x) x%2 -#define HEAD_ALGO(x) x/2 - -extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type); -extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type); -extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output); -extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize); -extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize); -extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output); -extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output); -extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output); +#define ALGO_SZ_LOSSY 1 // SZ compress + +#define HEAD_MODE(x) x % 2 +#define HEAD_ALGO(x) x / 2 + +extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type); +extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, + const char type); +extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize); +extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, + int32_t outputSize); +extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); // lossy -extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); -extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); -extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); -extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); +extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output); +extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output); +extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output); +extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output); #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; -// init call -int tsCompressInit(); -// exit call -void tsCompressExit(); +int32_t tsCompressInit(); +void tsCompressExit(); #endif -static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -90,8 +94,9 @@ static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize } } -static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -103,12 +108,13 @@ static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compres } } -static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -116,8 +122,9 @@ static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSiz } } -static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -129,12 +136,13 @@ static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compre } } -static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -142,8 +150,9 @@ static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, co } } -static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); } else if (algorithm == TWO_STAGE_COMP) { @@ -155,12 +164,13 @@ static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedS } } -static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -168,8 +178,9 @@ static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, } } -static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -181,12 +192,13 @@ static FORCE_INLINE int tsDecompressBigint(const char *const input, int compress } } -static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressBoolImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressBoolImp(input, nelements, buffer); + int32_t len = tsCompressBoolImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -194,8 +206,9 @@ static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, c } } -static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressBoolImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { @@ -207,47 +220,51 @@ static FORCE_INLINE int tsDecompressBool(const char *const input, int compressed } } -static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { return tsCompressStringImp(input, inputSize, output, outputSize); } -static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsDecompressStringImp(input, compressedSize, output, outputSize); } -static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { #ifdef TD_TSZ // lossy mode - if(lossyFloat) { + if (lossyFloat) { return tsCompressFloatLossyImp(input, nelements, output); - // lossless mode + // lossless mode } else { -#endif +#endif if (algorithm == ONE_STAGE_COMP) { return tsCompressFloatImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressFloatImp(input, nelements, buffer); + int32_t len = tsCompressFloatImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); return -1; - } -#ifdef TD_TSZ + } +#ifdef TD_TSZ } #endif } -static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { #ifdef TD_TSZ - if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); } else { -#endif +#endif // decompress lossless if (algorithm == ONE_STAGE_COMP) { return tsDecompressFloatImp(input, nelements, output); @@ -258,43 +275,44 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } #endif } - -static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { -#ifdef TD_TSZ - if(lossyDouble){ +static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { +#ifdef TD_TSZ + if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(input, nelements, output); } else { -#endif +#endif // lossless mode if (algorithm == ONE_STAGE_COMP) { return tsCompressDoubleImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressDoubleImp(input, nelements, buffer); + int32_t len = tsCompressDoubleImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } -#endif +#endif } -static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { - #ifdef TD_TSZ - if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ +static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } else { - #endif +#endif // decompress lossless if (algorithm == ONE_STAGE_COMP) { return tsDecompressDoubleImp(input, nelements, output); @@ -305,43 +323,48 @@ static FORCE_INLINE int tsDecompressDouble(const char *const input, int compress assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } -#endif +#endif } -#ifdef TD_TSZ +#ifdef TD_TSZ // // lossy float double // -static FORCE_INLINE int tsCompressFloatLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsCompressFloatLossyImp(input, nelements, output); } -static FORCE_INLINE int tsDecompressFloatLossy(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsDecompressFloatLossy(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); } -static FORCE_INLINE int tsCompressDoubleLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsCompressDoubleLossy(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsCompressDoubleLossyImp(input, nelements, output); } -static FORCE_INLINE int tsDecompressDoubleLossy(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } #endif -static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressTimestampImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressTimestampImp(input, nelements, buffer); + int32_t len = tsCompressTimestampImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -349,8 +372,9 @@ static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSi } } -static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressTimestampImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { @@ -366,4 +390,4 @@ static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compr } #endif -#endif /*_TD_UTIL_COMPRESSION_H*/ \ No newline at end of file +#endif /*_TD_UTIL_COMPRESSION_H_*/ \ No newline at end of file diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 11dd5414ec66eefc9cf1435ccbd3f51cad69e9f4..8275054a64f188c6f30a70a481f9ff9f9ab95d2b 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -17,7 +17,6 @@ #ifndef _TD_CONFIG_H_ #define _TD_CONFIG_H_ -#include "os.h" #include "tarray.h" #ifdef __cplusplus @@ -54,11 +53,11 @@ typedef struct SConfigItem { bool tsc; char *name; union { - bool bval; - float fval; - int32_t i32; - int64_t i64; - char *str; + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; }; union { int64_t imin; @@ -76,16 +75,17 @@ typedef struct { const char *value; } SConfigPair; -typedef struct SConfig SConfig; +typedef struct SConfig { + ECfgSrcType stype; + SArray *array; +} SConfig; SConfig *cfgInit(); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); -int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair void cfgCleanup(SConfig *pCfg); int32_t cfgGetSize(SConfig *pCfg); -SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter); -void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); diff --git a/include/util/tcrc32c.h b/include/util/tcrc32c.h index 200103b173d44d73d0c640588e521ba9ae6f0ca0..0246e09c31140aa2fe943de868b8056f320be883 100644 --- a/include/util/tcrc32c.h +++ b/include/util/tcrc32c.h @@ -18,8 +18,10 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_UTIL_CRC32_H -#define _TD_UTIL_CRC32_H +#ifndef _TD_UTIL_CRC32_H_ +#define _TD_UTIL_CRC32_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -39,4 +41,4 @@ void taosResolveCRC(); } #endif -#endif /*_TD_UTIL_CRC32_H*/ +#endif /*_TD_UTIL_CRC32_H_*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index 530f2b1b163d7102e461bb779f0d1d49c4f7e2c3..de5c8cda55d06b6d8f4aae2cb81e27201e51b3db 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -// clang-format off - -#ifndef _TD_UTIL_DEF_H -#define _TD_UTIL_DEF_H +#ifndef _TD_UTIL_DEF_H_ +#define _TD_UTIL_DEF_H_ #include "os.h" @@ -26,89 +24,106 @@ extern "C" { #define TSDB__packed -#define TSKEY int64_t -#define TSKEY_MIN INT64_MIN -#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY int64_t +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; // TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) -#define TSDB_KEYSIZE sizeof(TSKEY) -#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) // NULL definition -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL - -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF - -#define TSDB_DATA_UTINYINT_NULL 0xFF -#define TSDB_DATA_USMALLINT_NULL 0xFFFF -#define TSDB_DATA_UINT_NULL 0xFFFFFFFF -#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL - -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" - -#define TSDB_NETTEST_USER "nettestinternal" -#define TSDB_DEFAULT_USER "root" +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF + +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" + +#define TSDB_NETTEST_USER "nettestinternal" +#define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ -#define TSDB_DEFAULT_PASS "powerdb" +#define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) -#define TSDB_DEFAULT_PASS "tqueue" +#define TSDB_DEFAULT_PASS "tqueue" #elif (_TD_PRO_ == true) -#define TSDB_DEFAULT_PASS "prodb" +#define TSDB_DEFAULT_PASS "prodb" #else -#define TSDB_DEFAULT_PASS "taosdata" +#define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 +#define SHELL_MAX_PASSWORD_LEN 20 -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 #define TSDB_ERR -1 #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' -#define TSDB_TIME_PRECISION_MILLI 0 -#define TSDB_TIME_PRECISION_MICRO 1 -#define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" -#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) +#define TSDB_INFORMATION_SCHEMA_DB "information_schema" +#define TSDB_INS_TABLE_DNODES "dnodes" +#define TSDB_INS_TABLE_MNODES "mnodes" +#define TSDB_INS_TABLE_MODULES "modules" +#define TSDB_INS_TABLE_QNODES "qnodes" +#define TSDB_INS_TABLE_USER_DATABASE "user_database" +#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" +#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" +#define TSDB_INS_TABLE_USER_STABLES "user_stables" +#define TSDB_INS_TABLE_USER_STREAMS "user_streams" +#define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" +#define TSDB_INS_TABLE_USER_USERS "user_users" +#define TSDB_INS_TABLE_VGROUPS "vgroups" + +#define TSDB_TICK_PER_SECOND(precision) \ + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ -do {\ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ -} while(0) -#define T_READ_MEMBER(src, type, target) \ -do { \ - (target) = *(type *)(src); \ - (src) = (void *)((char *)src + sizeof(type));\ -} while(0) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ + do { \ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ + } while (0) +#define T_READ_MEMBER(src, type, target) \ + do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type)); \ + } while (0) typedef enum EOperatorType { // arithmetic operator @@ -149,194 +164,184 @@ typedef enum EOperatorType { OP_TYPE_JSON_CONTAINS } EOperatorType; - typedef enum ELogicConditionType { LOGIC_COND_TYPE_AND, LOGIC_COND_TYPE_OR, LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define FUNCTION_CEIL 4500 +#define FUNCTION_FLOOR 4501 +#define FUNCTION_ABS 4502 +#define FUNCTION_ROUND 4503 -#define FUNCTION_CEIL 4500 -#define FUNCTION_FLOOR 4501 -#define FUNCTION_ABS 4502 -#define FUNCTION_ROUND 4503 +#define FUNCTION_LENGTH 4800 +#define FUNCTION_CONCAT 4801 +#define FUNCTION_LTRIM 4802 +#define FUNCTION_RTRIM 4803 -#define FUNCTION_LENGTH 4800 -#define FUNCTION_CONCAT 4801 -#define FUNCTION_LTRIM 4802 -#define FUNCTION_RTRIM 4803 +#define TSDB_NAME_DELIMITER_LEN 1 -#define TSDB_NAME_DELIMITER_LEN 1 - -#define TSDB_UNI_LEN 24 -#define TSDB_USER_LEN TSDB_UNI_LEN +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 - -#define TSDB_MAX_COLUMNS 4096 -#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns - -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) - -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_COMMENT_LEN 4096 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 -#define TSDB_FUNC_TYPE_SCALAR 1 -#define TSDB_FUNC_TYPE_AGGREGATE 2 -#define TSDB_FUNC_MAX_RETRIEVE 1024 - -#define TSDB_TYPE_STR_MAX_LEN 32 -#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN -#define TSDB_CONSUMER_GROUP_LEN 192 -#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_COL_NAME_LEN 65 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 1024 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb - -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 - /** - * In some scenarios uint16_t (0~65535) is used to store the row len. - * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. - * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-(4096-1)*4 = 49151. - */ -#define TSDB_MAX_BYTES_PER_ROW 49151 -#define TSDB_MAX_TAGS_LEN 16384 -#define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 - -#define TSDB_AUTH_LEN 16 -#define TSDB_PASSWORD_LEN 32 -#define TSDB_USET_PASSWORD_LEN 129 -#define TSDB_VERSION_LEN 12 -#define TSDB_LABEL_LEN 8 - -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 -#define TSDB_SLOW_QUERY_SQL_LEN 512 - -#define TSDB_TRANS_STAGE_LEN 12 -#define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 64 - -#define TSDB_STEP_NAME_LEN 32 -#define TSDB_STEP_DESC_LEN 128 - -#define TSDB_ERROR_MSG_LEN 1024 -#define TSDB_DNODE_CONFIG_LEN 128 -#define TSDB_DNODE_VALUE_LEN 256 - -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 - -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 - -#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE +#define TSDB_ACCT_ID_LEN 11 + +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns + +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) + +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 + +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_CONSUMER_GROUP_LEN 192 +#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 1024 +#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb + +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_STB_COMMENT_LEN 1024 +/** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 + +#define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 +#define TSDB_VERSION_LEN 12 +#define TSDB_LABEL_LEN 8 + +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 + +#define TSDB_TRANS_STAGE_LEN 12 +#define TSDB_TRANS_TYPE_LEN 16 +#define TSDB_TRANS_ERROR_LEN 64 + +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + +#define TSDB_ERROR_MSG_LEN 1024 +#define TSDB_DNODE_CONFIG_LEN 128 +#define TSDB_DNODE_VALUE_LEN 256 + +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 + +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + +#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE -#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 -#define TSDB_DEFAULT_VN_PER_DB 2 +#define TSDB_DEFAULT_VN_PER_DB 2 -#define TSDB_DNODE_ROLE_ANY 0 -#define TSDB_DNODE_ROLE_MGMT 1 -#define TSDB_DNODE_ROLE_VNODE 2 +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) -#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 3650 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_KEEP 1 // data in db to be reserved. -#define TSDB_MAX_KEEP 365000 // data in db to be reserved. -#define TSDB_DEFAULT_KEEP 3650 // ten years +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 - -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 #define TSDB_MIN_DB_TTL_OPTION 1 #define TSDB_DEFAULT_DB_TTL_OPTION 0 @@ -352,75 +357,90 @@ typedef enum ELogicConditionType { #define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_MAX_FIELD_LEN 16384 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define PRIMARYKEY_TIMESTAMP_COL_ID 1 -#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_ID 1 +#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default +#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table - -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage - -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u -#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u -#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file -#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query - -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) - -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 - -#define TSDB_MAX_WAL_SIZE (1024*1024*3) - -#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P - -#define TFS_MAX_TIERS 3 +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage + +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query + +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_MAX_WAL_SIZE (1024 * 1024 * 3) + +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +#define TFS_MAX_TIERS 3 #define TFS_MAX_DISKS_PER_TIER 16 -#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 +#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; +typedef struct { + char dir[TSDB_FILENAME_LEN]; + int32_t level; + int32_t primary; +} SDiskCfg; + #ifdef __cplusplus } #endif diff --git a/include/util/tcfg.h b/include/util/tdes.h similarity index 77% rename from include/util/tcfg.h rename to include/util/tdes.h index 7ccfd4b0f5d9a739016f506e7a121276a1cac2a5..edb325da28ebdaa4085c1580fb320ea0bdd0a2b0 100644 --- a/include/util/tcfg.h +++ b/include/util/tdes.h @@ -13,23 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_CFG_H_ -#define _TD_COMMON_CFG_H_ +#ifndef _TD_UTIL_DES_H +#define _TD_UTIL_DES_H + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "tdef.h" - -typedef struct { - char dir[TSDB_FILENAME_LEN]; - int level; - int primary; -} SDiskCfg; +char *taosDesEncode(int64_t key, char *src, int32_t len); +char *taosDesDecode(int64_t key, char *src, int32_t len); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_CFG_H_*/ +#endif /*_TD_UTIL_DES_H*/ \ No newline at end of file diff --git a/include/util/encode.h b/include/util/tencode.h similarity index 58% rename from include/util/encode.h rename to include/util/tencode.h index ba63759737f250e120390dc1c93e676cbb345824..c058eebb5016141dc102aeed01621b7f4cb368fc 100644 --- a/include/util/encode.h +++ b/include/util/tencode.h @@ -16,8 +16,8 @@ #ifndef _TD_UTIL_ENCODE_H_ #define _TD_UTIL_ENCODE_H_ -#include "freelist.h" #include "tcoding.h" +#include "tfreelist.h" #include "tmacro.h" #ifdef __cplusplus @@ -71,62 +71,62 @@ typedef struct { TD_SLIST(SCoderNode) stack; } SCoder; -#define TD_CODER_POS(CODER) ((CODER)->pos) -#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) -#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) +#define TD_CODER_POS(CODER) ((CODER)->pos) +#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) +#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) +#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); /* ------------------------ ENCODE ------------------------ */ -int tStartEncode(SCoder* pEncoder); -void tEndEncode(SCoder* pEncoder); -static int tEncodeU8(SCoder* pEncoder, uint8_t val); -static int tEncodeI8(SCoder* pEncoder, int8_t val); -static int tEncodeU16(SCoder* pEncoder, uint16_t val); -static int tEncodeI16(SCoder* pEncoder, int16_t val); -static int tEncodeU32(SCoder* pEncoder, uint32_t val); -static int tEncodeI32(SCoder* pEncoder, int32_t val); -static int tEncodeU64(SCoder* pEncoder, uint64_t val); -static int tEncodeI64(SCoder* pEncoder, int64_t val); -static int tEncodeU16v(SCoder* pEncoder, uint16_t val); -static int tEncodeI16v(SCoder* pEncoder, int16_t val); -static int tEncodeU32v(SCoder* pEncoder, uint32_t val); -static int tEncodeI32v(SCoder* pEncoder, int32_t val); -static int tEncodeU64v(SCoder* pEncoder, uint64_t val); -static int tEncodeI64v(SCoder* pEncoder, int64_t val); -static int tEncodeFloat(SCoder* pEncoder, float val); -static int tEncodeDouble(SCoder* pEncoder, double val); -static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); -static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); -static int tEncodeCStr(SCoder* pEncoder, const char* val); +int32_t tStartEncode(SCoder* pEncoder); +void tEndEncode(SCoder* pEncoder); +static int32_t tEncodeU8(SCoder* pEncoder, uint8_t val); +static int32_t tEncodeI8(SCoder* pEncoder, int8_t val); +static int32_t tEncodeU16(SCoder* pEncoder, uint16_t val); +static int32_t tEncodeI16(SCoder* pEncoder, int16_t val); +static int32_t tEncodeU32(SCoder* pEncoder, uint32_t val); +static int32_t tEncodeI32(SCoder* pEncoder, int32_t val); +static int32_t tEncodeU64(SCoder* pEncoder, uint64_t val); +static int32_t tEncodeI64(SCoder* pEncoder, int64_t val); +static int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val); +static int32_t tEncodeI16v(SCoder* pEncoder, int16_t val); +static int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val); +static int32_t tEncodeI32v(SCoder* pEncoder, int32_t val); +static int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val); +static int32_t tEncodeI64v(SCoder* pEncoder, int64_t val); +static int32_t tEncodeFloat(SCoder* pEncoder, float val); +static int32_t tEncodeDouble(SCoder* pEncoder, double val); +static int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); +static int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); +static int32_t tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ -int tStartDecode(SCoder* pDecoder); -void tEndDecode(SCoder* pDecoder); -static bool tDecodeIsEnd(SCoder* pCoder); -static int tDecodeU8(SCoder* pDecoder, uint8_t* val); -static int tDecodeI8(SCoder* pDecoder, int8_t* val); -static int tDecodeU16(SCoder* pDecoder, uint16_t* val); -static int tDecodeI16(SCoder* pDecoder, int16_t* val); -static int tDecodeU32(SCoder* pDecoder, uint32_t* val); -static int tDecodeI32(SCoder* pDecoder, int32_t* val); -static int tDecodeU64(SCoder* pDecoder, uint64_t* val); -static int tDecodeI64(SCoder* pDecoder, int64_t* val); -static int tDecodeU16v(SCoder* pDecoder, uint16_t* val); -static int tDecodeI16v(SCoder* pDecoder, int16_t* val); -static int tDecodeU32v(SCoder* pDecoder, uint32_t* val); -static int tDecodeI32v(SCoder* pDecoder, int32_t* val); -static int tDecodeU64v(SCoder* pDecoder, uint64_t* val); -static int tDecodeI64v(SCoder* pDecoder, int64_t* val); -static int tDecodeFloat(SCoder* pDecoder, float* val); -static int tDecodeDouble(SCoder* pDecoder, double* val); -static int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); -static int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); -static int tDecodeCStr(SCoder* pDecoder, const char** val); -static int tDecodeCStrTo(SCoder* pDecoder, char* val); +int32_t tStartDecode(SCoder* pDecoder); +void tEndDecode(SCoder* pDecoder); +static bool tDecodeIsEnd(SCoder* pCoder); +static int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val); +static int32_t tDecodeI8(SCoder* pDecoder, int8_t* val); +static int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val); +static int32_t tDecodeI16(SCoder* pDecoder, int16_t* val); +static int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val); +static int32_t tDecodeI32(SCoder* pDecoder, int32_t* val); +static int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val); +static int32_t tDecodeI64(SCoder* pDecoder, int64_t* val); +static int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val); +static int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val); +static int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val); +static int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val); +static int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val); +static int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val); +static int32_t tDecodeFloat(SCoder* pDecoder, float* val); +static int32_t tDecodeDouble(SCoder* pDecoder, double* val); +static int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); +static int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); +static int32_t tDecodeCStr(SCoder* pDecoder, const char** val); +static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val); /* ------------------------ IMPL ------------------------ */ #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ @@ -190,7 +190,7 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val); return 0; // 8 -static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) { +static FORCE_INLINE int32_t tEncodeU8(SCoder* pEncoder, uint8_t val) { if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val); @@ -199,7 +199,7 @@ static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) { return 0; } -static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) { +static FORCE_INLINE int32_t tEncodeI8(SCoder* pEncoder, int8_t val) { if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(int8_t, TD_CODER_CURRENT(pEncoder), val); @@ -209,31 +209,31 @@ static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) { } // 16 -static FORCE_INLINE int tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); } -static FORCE_INLINE int tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); } +static FORCE_INLINE int32_t tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); } -static FORCE_INLINE int tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); } +static FORCE_INLINE int32_t tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); } -static FORCE_INLINE int tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); } +static FORCE_INLINE int32_t tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI16v(SCoder* pEncoder, int16_t val) { +static FORCE_INLINE int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI16v(SCoder* pEncoder, int16_t val) { return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val)); } // 32v -static FORCE_INLINE int tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI32v(SCoder* pEncoder, int32_t val) { +static FORCE_INLINE int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI32v(SCoder* pEncoder, int32_t val) { return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val)); } // 64v -static FORCE_INLINE int tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI64v(SCoder* pEncoder, int64_t val) { +static FORCE_INLINE int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI64v(SCoder* pEncoder, int64_t val) { return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val)); } -static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { +static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) { union { uint32_t ui; float f; @@ -242,7 +242,7 @@ static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { return tEncodeU32(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { +static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) { union { uint64_t ui; double d; @@ -251,7 +251,7 @@ static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { return tEncodeU64(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) { +static FORCE_INLINE int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) { if (tEncodeU64v(pEncoder, len) < 0) return -1; if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1; @@ -262,24 +262,24 @@ static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_ return 0; } -static FORCE_INLINE int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { +static FORCE_INLINE int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { return tEncodeBinary(pEncoder, (void*)val, len + 1); } -static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { +static FORCE_INLINE int32_t tEncodeCStr(SCoder* pEncoder, const char* val) { return tEncodeCStrWithLen(pEncoder, val, (uint64_t)strlen(val)); } /* ------------------------ FOR DECODER ------------------------ */ // 8 -static FORCE_INLINE int tDecodeU8(SCoder* pDecoder, uint8_t* val) { +static FORCE_INLINE int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val) { if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); return 0; } -static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) { +static FORCE_INLINE int32_t tDecodeI8(SCoder* pDecoder, int8_t* val) { if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); @@ -287,21 +287,27 @@ static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) { } // 16 -static FORCE_INLINE int tDecodeU16(SCoder* pDecoder, uint16_t* val) { TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); } -static FORCE_INLINE int tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); +} +static FORCE_INLINE int32_t tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tDecodeU32(SCoder* pDecoder, uint32_t* val) { TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); } -static FORCE_INLINE int tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); +} +static FORCE_INLINE int32_t tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tDecodeU64(SCoder* pDecoder, uint64_t* val) { TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); } -static FORCE_INLINE int tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); +} +static FORCE_INLINE int32_t tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tDecodeU16v(SCoder* pDecoder, uint16_t* val) { +static FORCE_INLINE int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint16_t); } -static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) { +static FORCE_INLINE int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val) { uint16_t tval; if (tDecodeU16v(pDecoder, &tval) < 0) { return -1; @@ -311,11 +317,11 @@ static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) { } // 32v -static FORCE_INLINE int tDecodeU32v(SCoder* pDecoder, uint32_t* val) { +static FORCE_INLINE int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint32_t); } -static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) { +static FORCE_INLINE int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val) { uint32_t tval; if (tDecodeU32v(pDecoder, &tval) < 0) { return -1; @@ -325,11 +331,11 @@ static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) { } // 64v -static FORCE_INLINE int tDecodeU64v(SCoder* pDecoder, uint64_t* val) { +static FORCE_INLINE int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint64_t); } -static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { +static FORCE_INLINE int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val) { uint64_t tval; if (tDecodeU64v(pDecoder, &tval) < 0) { return -1; @@ -338,7 +344,7 @@ static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { return 0; } -static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { +static FORCE_INLINE int32_t tDecodeFloat(SCoder* pDecoder, float* val) { union { uint32_t ui; float f; @@ -352,7 +358,7 @@ static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { return 0; } -static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { +static FORCE_INLINE int32_t tDecodeDouble(SCoder* pDecoder, double* val) { union { uint64_t ui; double d; @@ -366,7 +372,7 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { return 0; } -static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { +static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; @@ -376,18 +382,18 @@ static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64 return 0; } -static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { +static FORCE_INLINE int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { if (tDecodeBinary(pDecoder, (const void**)val, len) < 0) return -1; (*len) -= 1; return 0; } -static FORCE_INLINE int tDecodeCStr(SCoder* pDecoder, const char** val) { +static FORCE_INLINE int32_t tDecodeCStr(SCoder* pDecoder, const char** val) { uint64_t len; return tDecodeCStrAndLen(pDecoder, val, &len); } -static int tDecodeCStrTo(SCoder* pDecoder, char* val) { +static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val) { const char* pStr; uint64_t len; if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1; diff --git a/include/util/texception.h b/include/util/texception.h new file mode 100644 index 0000000000000000000000000000000000000000..576545d96c3641499082c0d86d8fb689829726d9 --- /dev/null +++ b/include/util/texception.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2020 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_EXCEPTION_H_ +#define _TD_UTIL_EXCEPTION_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * cleanup actions + */ +typedef struct SCleanupAction { + bool failOnly; + uint8_t wrapper; + uint16_t reserved; + void* func; + union { + void* Ptr; + bool Bool; + char Char; + int8_t Int8; + uint8_t Uint8; + int16_t Int16; + uint16_t Uint16; + int32_t Int; + uint32_t Uint; + int32_t Int32; + uint32_t Uint32; + int64_t Int64; + uint64_t Uint64; + float Float; + double Double; + } arg1, arg2; +} SCleanupAction; + +/* + * exception hander registration + */ +typedef struct SExceptionNode { + struct SExceptionNode* prev; + jmp_buf jb; + int32_t code; + int32_t maxCleanupAction; + int32_t numCleanupAction; + SCleanupAction* cleanupActions; +} SExceptionNode; + +// functions & macros for auto-cleanup + +void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2); +void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2); +void cleanupPush_void_ptr(bool failOnly, void* func, void* arg); +void cleanupPush_int_int(bool failOnly, void* func, int32_t arg); +void cleanupPush_void(bool failOnly, void* func); +void cleanupPush_int_ptr(bool failOnly, void* func, void* arg); + +int32_t cleanupGetActionCount(); +void cleanupExecuteTo(int32_t anchor, bool failed); +void cleanupExecute(SExceptionNode* node, bool failed); +bool cleanupExceedLimit(); + +#define CLEANUP_PUSH_VOID_PTR_PTR(failOnly, func, arg1, arg2) \ + cleanupPush_void_ptr_ptr((failOnly), (void*)(func), (void*)(arg1), (void*)(arg2)) +#define CLEANUP_PUSH_VOID_PTR_BOOL(failOnly, func, arg1, arg2) \ + cleanupPush_void_ptr_bool((failOnly), (void*)(func), (void*)(arg1), (bool)(arg2)) +#define CLEANUP_PUSH_VOID_PTR(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (void*)(arg)) +#define CLEANUP_PUSH_INT_INT(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (int32_t)(arg)) +#define CLEANUP_PUSH_VOID(failOnly, func) cleanupPush_void((failOnly), (void*)(func)) +#define CLEANUP_PUSH_INT_PTR(failOnly, func, arg) cleanupPush_int_ptr((failOnly), (void*)(func), (void*)(arg)) +#define CLEANUP_PUSH_FREE(failOnly, arg) cleanupPush_void_ptr((failOnly), free, (void*)(arg)) +#define CLEANUP_PUSH_CLOSE(failOnly, arg) cleanupPush_int_int((failOnly), close, (int32_t)(arg)) +#define CLEANUP_PUSH_FCLOSE(failOnly, arg) cleanupPush_int_ptr((failOnly), fclose, (void*)(arg)) + +#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() +#define CLEANUP_EXECUTE_TO(anchor, failed) cleanupExecuteTo((anchor), (failed)) +#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit() + +// functions & macros for exception handling + +void exceptionPushNode(SExceptionNode* node); +int32_t exceptionPopNode(); +void exceptionThrow(int32_t code); + +#define TRY(maxCleanupActions) \ + do { \ + SExceptionNode exceptionNode = {0}; \ + SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \ + exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \ + exceptionNode.cleanupActions = cleanupActions; \ + exceptionPushNode(&exceptionNode); \ + int32_t caughtException = setjmp(exceptionNode.jb); \ + if (caughtException == 0) + +#define CATCH(code) \ + int32_t code = exceptionPopNode(); \ + if (caughtException == 1) + +#define FINALLY(code) int32_t code = exceptionPopNode(); + +#define END_TRY \ + } \ + while (0) \ + ; + +#define THROW(x) exceptionThrow((x)) +#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1)) +#define CLEANUP_EXECUTE() cleanupExecute(&exceptionNode, CAUGHT_EXCEPTION()) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_EXCEPTION_H_*/ diff --git a/include/util/tfile.h b/include/util/tfile.h deleted file mode 100644 index 59953de861f0f123dad05bf8cde1dd4ee70600f7..0000000000000000000000000000000000000000 --- a/include/util/tfile.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_FILE_H -#define _TD_UTIL_FILE_H - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// init taos file module -int32_t tfInit(); - -// clean up taos file module -void tfCleanup(); - -// the same syntax as UNIX standard open/close/read/write -// but FD is int64_t and will never be reused -// int64_t tfOpenRead(const char *pathname); -// int64_t tfOpenReadWrite(const char *pathname); -// int64_t tfOpenCreateWrite(const char *pathname); -// int64_t tfOpenCreateWriteAppend(const char *pathname); - -// int64_t tfClose(int64_t tfd); -// int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -// int64_t tfRead(int64_t tfd, void *buf, int64_t count); -// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -// int32_t tfFsync(int64_t tfd); -// bool tfValid(int64_t tfd); -// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -// int32_t tfFtruncate(int64_t tfd, int64_t length); -// void * tfMmapReadOnly(int64_t tfd, int64_t length); -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_FILE_H*/ diff --git a/include/util/freelist.h b/include/util/tfreelist.h similarity index 97% rename from include/util/freelist.h rename to include/util/tfreelist.h index 497a6d58c3f0791a90765f0682f5474ca8a2b6cf..c1913ebaf95866218db3d6b151a903f040e2175e 100644 --- a/include/util/freelist.h +++ b/include/util/tfreelist.h @@ -16,7 +16,6 @@ #ifndef _TD_UTIL_FREELIST_H_ #define _TD_UTIL_FREELIST_H_ -#include "os.h" #include "tlist.h" #ifdef __cplusplus diff --git a/include/util/tfunctional.h b/include/util/tfunctional.h index c96f997f06c363bfb06b8177a50b801a0cfbaea7..43e3cd5e483a37bad242a75799a836e5433012ce 100644 --- a/include/util/tfunctional.h +++ b/include/util/tfunctional.h @@ -12,45 +12,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_FUNCTIONAL_H -#define _TD_UTIL_FUNCTIONAL_H +#ifndef _TD_UTIL_FUNCTIONAL_H_ +#define _TD_UTIL_FUNCTIONAL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -//TODO: hard to use, trying to rewrite it using va_list +// TODO: hard to use, trying to rewrite it using va_list typedef void* (*GenericVaFunc)(void* args[]); -typedef int32_t (*I32VaFunc) (void* args[]); -typedef void (*VoidVaFunc) (void* args[]); +typedef int32_t (*I32VaFunc)(void* args[]); +typedef void (*VoidVaFunc)(void* args[]); typedef struct GenericSavedFunc { GenericVaFunc func; - void * args[]; + void* args[]; } tGenericSavedFunc; typedef struct I32SavedFunc { I32VaFunc func; - void * args[]; + void* args[]; } tI32SavedFunc; typedef struct VoidSavedFunc { VoidVaFunc func; - void * args[]; + void* args[]; } tVoidSavedFunc; -tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs); -tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs); -tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs); -void* genericInvoke(tGenericSavedFunc* const pSavedFunc); -int32_t i32Invoke(tI32SavedFunc* const pSavedFunc); -void voidInvoke(tVoidSavedFunc* const pSavedFunc); +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs); +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs); +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs); +void* genericInvoke(tGenericSavedFunc* const pSavedFunc); +int32_t i32Invoke(tI32SavedFunc* const pSavedFunc); +void voidInvoke(tVoidSavedFunc* const pSavedFunc); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_FUNCTIONAL_H*/ +#endif /*_TD_UTIL_FUNCTIONAL_H_*/ diff --git a/include/util/thash.h b/include/util/thash.h index 1f72045a830e150e555f66fe36e0d453e680eff6..5c344f3f0fc5e59cafca0c461639cab9ae052225 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -13,22 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_HASH_H -#define TDENGINE_HASH_H +#ifndef _TD_UTIL_HASH_H_ +#define _TD_UTIL_HASH_H_ + +#include "tarray.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif -#include "tarray.h" -#include "tlockfree.h" - typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); -typedef int32_t (*_equal_fn_t)(const void*, const void*, size_t len); +typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_INDEX(v, c) ((v) & ((c)-1)) @@ -59,43 +59,43 @@ _equal_fn_t taosGetDefaultEqualFunction(int32_t type); typedef struct SHashNode { struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t count; // reference count - int8_t removed; // flag to indicate removed + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t count; // reference count + int8_t removed; // flag to indicate removed char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) +#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) typedef enum SHashLockTypeE { - HASH_NO_LOCK = 0, - HASH_ENTRY_LOCK = 1, + HASH_NO_LOCK = 0, + HASH_ENTRY_LOCK = 1, } SHashLockTypeE; typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch SHashNode *next; } SHashEntry; typedef struct SHashObj { - SHashEntry **hashList; - uint32_t capacity; // number of slots - uint32_t size; // number of elements in hash table - - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function - _equal_fn_t equalFp; // equal function - _hash_before_fn_t callbackFp; // function invoked before return the value to caller - - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry + SHashEntry **hashList; + uint32_t capacity; // number of slots + uint32_t size; // number of elements in hash table + + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + _equal_fn_t equalFp; // equal function + _hash_before_fn_t callbackFp; // function invoked before return the value to caller + + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; /** @@ -128,7 +128,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded); - /** * return the payload data with the specified key * @@ -147,7 +146,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param destBuf * @return */ -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* destBuf); +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); /** * Clone the result to interval allocated buffer @@ -157,7 +156,7 @@ void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* * @param destBuf * @return */ -void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz); +void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); /** * remove item with the specified key @@ -206,15 +205,14 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); * @param pHashObj * @param p */ -void taosHashCancelIterate(SHashObj *pHashObj, void *p); +void taosHashCancelIterate(SHashObj *pHashObj, void *p); /** * Get the corresponding key information for a given data in hash table * @param data * @return */ -int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); - +int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); /** * Get the corresponding key information for a given data in hash table, using memcpy @@ -222,13 +220,13 @@ int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); * @param dst * @return */ -static FORCE_INLINE int32_t taosHashCopyKey(void *data, void* dst) { +static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) { if (NULL == data || NULL == dst) { return -1; } - - SHashNode * node = GET_HASH_PNODE(data); - void* key = GET_HASH_NODE_KEY(node); + + SHashNode *node = GET_HASH_PNODE(data); + void *key = GET_HASH_NODE_KEY(node); memcpy(dst, key, node->keyLen); return 0; @@ -249,7 +247,7 @@ int32_t taosHashGetDataLen(void *data); * @param keyLen * @return */ -void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); /** * release the prevous acquired obj @@ -262,9 +260,8 @@ void taosHashRelease(SHashObj *pHashObj, void *p); void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); - #ifdef __cplusplus } #endif -#endif // TDENGINE_HASH_H +#endif // _TD_UTIL_HASH_H_ diff --git a/include/util/theap.h b/include/util/theap.h index fd1a39f8dd654338edf219531eaf982c0898df79..fb5ff8301a9b08b2cb53c353a363042c2de9cc34 100644 --- a/include/util/theap.h +++ b/include/util/theap.h @@ -12,19 +12,20 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_HEAP_H -#define TDENGINE_HEAP_H + +#ifndef _TD_UTIL_HEAP_H_ +#define _TD_UTIL_HEAP_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - struct HeapNode; /* Return non-zero if a < b. */ -typedef int (*HeapCompareFn)(const struct HeapNode* a, const struct HeapNode* b); +typedef int32_t (*HeapCompareFn)(const struct HeapNode* a, const struct HeapNode* b); typedef struct HeapNode { struct HeapNode* left; @@ -38,15 +39,14 @@ typedef struct HeapNode { * */ typedef struct { - HeapNode* min; - size_t nelts; - HeapCompareFn compFn; + HeapNode* min; + size_t nelts; + HeapCompareFn compFn; } Heap; - Heap* heapCreate(HeapCompareFn fn); -void heapDestroy(Heap *heap); +void heapDestroy(Heap* heap); HeapNode* heapMin(const Heap* heap); @@ -56,10 +56,10 @@ void heapRemove(Heap* heap, struct HeapNode* node); void heapDequeue(Heap* heap); -size_t heapSize(Heap *heap); +size_t heapSize(Heap* heap); #ifdef __cplusplus } #endif -#endif // TDENGINE_HASH_H +#endif /*_TD_UTIL_HEAP_H_*/ diff --git a/include/util/tidpool.h b/include/util/tidpool.h index 1a977fd04c27b14b3ce0ce242b3d523e1d67d688..8596b439e3f24d684bbabc4c7364f609119d224d 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -13,31 +13,34 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_IDPOOL_H -#define _TD_UTIL_IDPOOL_H +#ifndef _TD_UTIL_IDPOOL_H_ +#define _TD_UTIL_IDPOOL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -void *taosInitIdPool(int maxId); - -int taosUpdateIdPool(void *handle, int maxId); - -int taosIdPoolMaxSize(void *handle); - -int taosAllocateId(void *handle); - -void taosFreeId(void *handle, int id); - -void taosIdPoolCleanUp(void *handle); - -int taosIdPoolNumOfUsed(void *handle); - -bool taosIdPoolMarkStatus(void *handle, int id); +typedef struct { + int32_t maxId; + int32_t numOfFree; + int32_t freeSlot; + bool *freeList; + pthread_mutex_t mutex; +} id_pool_t; + +void *taosInitIdPool(int32_t maxId); +int32_t taosUpdateIdPool(id_pool_t *handle, int32_t maxId); +int32_t taosIdPoolMaxSize(id_pool_t *handle); +int32_t taosAllocateId(id_pool_t *handle); +void taosFreeId(id_pool_t *handle, int32_t id); +void taosIdPoolCleanUp(id_pool_t *handle); +int32_t taosIdPoolNumOfUsed(id_pool_t *handle); +bool taosIdPoolMarkStatus(id_pool_t *handle, int32_t id); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_IDPOOL_H*/ +#endif /*_TD_UTIL_IDPOOL_H_*/ diff --git a/include/util/tjson.h b/include/util/tjson.h index e42e40efa7faed3ab52d5b5e18c2f88d54032f00..2d9f433ab2a89fc2f4613a50a81eb3c3bfe613c9 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -16,18 +16,18 @@ #ifndef _TD_UTIL_JSON_H_ #define _TD_UTIL_JSON_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" - typedef void SJson; SJson* tjsonCreateObject(); -void tjsonDelete(SJson* pJson); +void tjsonDelete(SJson* pJson); -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number); int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number); int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean); @@ -35,7 +35,7 @@ int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); -SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName); +SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName); int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal); int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal); int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal); @@ -48,7 +48,7 @@ int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); int32_t tjsonGetArraySize(const SJson* pJson); -SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index); +SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index); typedef int32_t (*FToJson)(const void* pObj, SJson* pJson); diff --git a/include/util/tkey.h b/include/util/tkey.h deleted file mode 100644 index 197cfff958416dc49db20b59dc8a2b6fe3216200..0000000000000000000000000000000000000000 --- a/include/util/tkey.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_UTIL_KEY_H -#define _TD_UTIL_KEY_H - -#ifdef __cplusplus -extern "C" { -#endif - -unsigned char *base64_decode(const char *value, int inlen, int *outlen); -char * base64_encode(const unsigned char *value, int vlen); -char * taosDesEncode(int64_t key, char *src, int len); -char * taosDesDecode(int64_t key, char *src, int len); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_KEY_H*/ \ No newline at end of file diff --git a/include/util/tlist.h b/include/util/tlist.h index 134873a993f3721e860ec443da281e2443dd04a1..caa642491898243a79509338447452bcd6dd31fa 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -12,8 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_LIST_H -#define _TD_UTIL_LIST_H + +#ifndef _TD_UTIL_LIST_H_ +#define _TD_UTIL_LIST_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -28,12 +31,12 @@ extern "C" { #define TD_SLIST(TYPE) \ struct { \ struct TYPE *sl_head_; \ - int sl_neles_; \ + int32_t sl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) -#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) #define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ @@ -79,16 +82,16 @@ extern "C" { struct { \ struct TYPE *dl_head_; \ struct TYPE *dl_tail_; \ - int dl_neles_; \ + int32_t dl_neles_; \ } -#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) -#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) #define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) #define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) -#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) -#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) -#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) #define TD_DLIST_INIT(dl) \ do { \ @@ -200,29 +203,29 @@ typedef struct SListNode { typedef struct { TD_DLIST(SListNode); - int eleSize; + int32_t eleSize; } SList; typedef struct { - SListNode * next; + SListNode *next; TD_LIST_DIRECTION_T direction; } SListIter; -#define listHead(l) TD_DLIST_HEAD(l) -#define listTail(l) TD_DLIST_TAIL(l) -#define listNEles(l) TD_DLIST_NELES(l) -#define listEleSize(l) ((l)->eleSize) -#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) +#define listHead(l) TD_DLIST_HEAD(l) +#define listTail(l) TD_DLIST_TAIL(l) +#define listNEles(l) TD_DLIST_NELES(l) +#define listEleSize(l) ((l)->eleSize) +#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) #define listNodeFree(n) free(n) -void tdListInit(SList *list, int eleSize); +void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); -SList * tdListNew(int eleSize); -void * tdListFree(SList *list); +SList *tdListNew(int32_t eleSize); +void *tdListFree(SList *list); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); -int tdListPrepend(SList *list, void *data); -int tdListAppend(SList *list, void *data); +int32_t tdListPrepend(SList *list, void *data); +int32_t tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); SListNode *tdListGetHead(SList *list); @@ -239,4 +242,4 @@ SListNode *tdListNext(SListIter *pIter); } #endif -#endif /*_TD_UTIL_LIST_H*/ \ No newline at end of file +#endif /*_TD_UTIL_LIST_H_*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index fec346cd0290b914df038d3a965603b8b27b3fad..e49e89c309a99238b28535d9621ff20c93c66573 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -12,8 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOCK_FREE_H -#define _TD_UTIL_LOCK_FREE_H + +#ifndef _TD_UTIL_LOCK_FREE_H_ +#define _TD_UTIL_LOCK_FREE_H_ #include "os.h" @@ -22,7 +23,7 @@ extern "C" { #endif // reference counting -typedef void (*_ref_fn_t)(const void* pObj); +typedef void (*_ref_fn_t)(const void *pObj); #define T_REF_DECLARE() \ struct { \ @@ -67,8 +68,6 @@ typedef void (*_ref_fn_t)(const void* pObj); #define T_REF_VAL_GET(x) (x)->_ref.val - - // single writer multiple reader lock typedef volatile int32_t SRWLatch; @@ -78,35 +77,33 @@ void taosWUnLockLatch(SRWLatch *pLatch); void taosRLockLatch(SRWLatch *pLatch); void taosRUnLockLatch(SRWLatch *pLatch); - - // copy on read -#define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \ +#define taosCorBeginRead(x) \ + for (uint32_t i_ = 1; 1; ++i_) { \ int32_t old_ = atomic_add_fetch_32((x), 0); \ - if (old_ & 0x00000001) { \ - if (i_ % 1000 == 0) { \ - sched_yield(); \ - } \ - continue; \ + if (old_ & 0x00000001) { \ + if (i_ % 1000 == 0) { \ + sched_yield(); \ + } \ + continue; \ } -#define taosCorEndRead(x) \ - if (atomic_add_fetch_32((x), 0) == old_) { \ - break; \ - } \ +#define taosCorEndRead(x) \ + if (atomic_add_fetch_32((x), 0) == old_) { \ + break; \ + } \ } -#define taosCorBeginWrite(x) taosCorBeginRead(x) \ - if (atomic_val_compare_exchange_32((x), old_, old_ + 1) != old_) { \ - continue; \ - } +#define taosCorBeginWrite(x) \ + taosCorBeginRead(x) if (atomic_val_compare_exchange_32((x), old_, old_ + 1) != old_) { continue; } -#define taosCorEndWrite(x) atomic_add_fetch_32((x), 1); \ - break; \ +#define taosCorEndWrite(x) \ + atomic_add_fetch_32((x), 1); \ + break; \ } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_LOCK_FREE_H*/ +#endif /*_TD_UTIL_LOCK_FREE_H_*/ diff --git a/include/util/tlog.h b/include/util/tlog.h index 1c14cc445f341992d176ba8a5387b3cdd0c93027..6e6795e9a2c6c564e8d5dc4fe0bd642401907141 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOG_H -#define _TD_UTIL_LOG_H +#ifndef _TD_UTIL_LOG_H_ +#define _TD_UTIL_LOG_H_ #include "os.h" @@ -84,4 +84,4 @@ extern int8_t tscEmbeddedInUtil; } #endif -#endif /*_TD_UTIL_LOG_H*/ +#endif /*_TD_UTIL_LOG_H_*/ diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index 241647ba1eff40c2860368923008432768f92485..51906443f5ab874a2e7b16a11304ed6890f90437 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -13,14 +13,16 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOSERTREE_H -#define _TD_UTIL_LOSERTREE_H +#ifndef _TD_UTIL_LOSERTREE_H_ +#define _TD_UTIL_LOSERTREE_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param); +typedef int32_t (*__merge_compare_fn_t)(const void *, const void *, void *param); typedef struct STreeNode { int32_t index; @@ -31,16 +33,17 @@ typedef struct SMultiwayMergeTreeInfo { int32_t numOfSources; int32_t totalSources; __merge_compare_fn_t comparFn; - void * param; + void *param; struct STreeNode *pNode; } SMultiwayMergeTreeInfo; #define tMergeTreeGetChosenIndex(t_) ((t_)->pNode[0].index) #define tMergeTreeGetAdjustIndex(t_) (tMergeTreeGetChosenIndex(t_) + (t_)->numOfSources) -int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, + __merge_compare_fn_t compareFn); -void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree); +void tMergeTreeDestroy(SMultiwayMergeTreeInfo *pTree); void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); @@ -52,4 +55,4 @@ void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree); } #endif -#endif /*_TD_UTIL_LOSERTREE_H*/ +#endif /*_TD_UTIL_LOSERTREE_H_*/ diff --git a/include/util/tmacro.h b/include/util/tmacro.h index 297c37d62a99763462a7a61222ababf706a62b43..07c6e6509e4d5eedd7c14a830922b8347111f6e8 100644 --- a/include/util/tmacro.h +++ b/include/util/tmacro.h @@ -24,7 +24,7 @@ extern "C" { // Module init/clear MACRO definitions #define TD_MOD_UNINITIALIZED 0 -#define TD_MOD_INITIALIZED 1 +#define TD_MOD_INITIALIZED 1 typedef int8_t td_mode_flag_t; diff --git a/include/util/mallocator.h b/include/util/tmallocator.h similarity index 85% rename from include/util/mallocator.h rename to include/util/tmallocator.h index 5ecdc316a490aa1be7b57ec68d00b7653cebf864..e9eb3e1b727755fdcf34f5b9c78771362d6a0b6e 100644 --- a/include/util/mallocator.h +++ b/include/util/tmallocator.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MALLOCATOR_H_ -#define _TD_MALLOCATOR_H_ +#ifndef _TD_UTIL_MALLOCATOR_H_ +#define _TD_UTIL_MALLOCATOR_H_ #include "os.h" @@ -29,10 +29,10 @@ extern "C" { void (*free_)(struct TYPE *, void *ptr); \ } #define TD_MA_MALLOC_FUNC(TMA) (TMA)->malloc_ -#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ +#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ #define TD_MA_MALLOC(TMA, SIZE) (*((TMA)->malloc_))(TMA, (SIZE)) -#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) +#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) typedef struct SMemAllocator { void *impl; @@ -40,7 +40,7 @@ typedef struct SMemAllocator { } SMemAllocator; #define tMalloc(pMA, SIZE) TD_MA_MALLOC(PMA, SIZE) -#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR) +#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR) typedef struct SMemAllocatorFactory { void *impl; @@ -52,4 +52,4 @@ typedef struct SMemAllocatorFactory { } #endif -#endif /*_TD_MALLOCATOR_H_*/ \ No newline at end of file +#endif /*_TD_UTIL_MALLOCATOR_H_*/ \ No newline at end of file diff --git a/include/util/tmd5.h b/include/util/tmd5.h index f8114ad57bb1a863d5e580b3028439dbb86f9e0f..6f88d95e24195b5c5c77c53afad19c4503010c9b 100644 --- a/include/util/tmd5.h +++ b/include/util/tmd5.h @@ -25,6 +25,12 @@ #ifndef _TD_UTIL_MD5_H #define _TD_UTIL_MD5_H +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { uint32_t i[2]; /* number of _bits_ handled mod 2^64 */ uint32_t buf[4]; /* scratch buffer */ @@ -33,7 +39,11 @@ typedef struct { } T_MD5_CTX; void tMD5Init(T_MD5_CTX *mdContext); -void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen); +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, uint32_t inLen); void tMD5Final(T_MD5_CTX *mdContext); +#ifdef __cplusplus +} +#endif + #endif /*_TD_UTIL_MD5_H*/ diff --git a/include/util/tmempool.h b/include/util/tmempool.h index 3e3db738a9f372082d785bedb28da776c23ba59f..7a5aca7b34f6393c5440f030f3c416ca18b08a0f 100644 --- a/include/util/tmempool.h +++ b/include/util/tmempool.h @@ -12,25 +12,24 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_MEMPOOL_H -#define _TD_UTIL_MEMPOOL_H +#ifndef _TD_UTIL_MEMPOOL_H_ +#define _TD_UTIL_MEMPOOL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#define mpool_h void * - -mpool_h taosMemPoolInit(int maxNum, int blockSize); - -char *taosMemPoolMalloc(mpool_h handle); - -void taosMemPoolFree(mpool_h handle, char *p); +typedef void *mpool_h; -void taosMemPoolCleanUp(mpool_h handle); +mpool_h taosMemPoolInit(int32_t maxNum, int32_t blockSize); +char *taosMemPoolMalloc(mpool_h handle); +void taosMemPoolFree(mpool_h handle, char *p); +void taosMemPoolCleanUp(mpool_h handle); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_MEMPOOL_H*/ +#endif /*_TD_UTIL_MEMPOOL_H_*/ diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index e989c31cd63f45cf0a9ac8c082f3cdb31f46436c..ce9a57c2c3ea22fc1c4b9ef9f8fbf00a73e137e5 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TPAGEDBUF_H -#define TDENGINE_TPAGEDBUF_H +#ifndef _TD_UTIL_PAGEDBUF_H_ +#define _TD_UTIL_PAGEDBUF_H_ + +#include "thash.h" +#include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif -#include "tlist.h" -#include "thash.h" -#include "os.h" -#include "tlockfree.h" - -typedef struct SArray* SIDList; -typedef struct SPageInfo SPageInfo; +typedef struct SArray* SIDList; +typedef struct SPageInfo SPageInfo; typedef struct SDiskbasedBuf SDiskbasedBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes -#define DEFAULT_PAGE_SIZE (16384L) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes typedef struct SFilePage { - int64_t num; + int32_t num; char data[]; } SFilePage; @@ -55,7 +53,7 @@ typedef struct SDiskbasedBufStatis { * @param handle * @return */ -int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); /** * @@ -64,7 +62,7 @@ int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t in * @param pageId * @return */ -SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId); +void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId); /** * @@ -80,7 +78,7 @@ SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId); * @param id * @return */ -SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id); +void* getBufPage(SDiskbasedBuf* pBuf, int32_t id); /** * release the referenced buf pages @@ -108,13 +106,13 @@ size_t getTotalBufSize(const SDiskbasedBuf* pBuf); * @param pBuf * @return */ -size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf); +size_t getNumOfBufGroupId(const SDiskbasedBuf* pBuf); /** * destroy result buffer * @param pBuf */ -void destroyResultBuf(SDiskbasedBuf* pBuf); +void destroyDiskbasedBuf(SDiskbasedBuf* pBuf); /** * @@ -137,6 +135,11 @@ int32_t getPageId(const SPageInfo* pPgInfo); */ int32_t getBufPageSize(const SDiskbasedBuf* pBuf); +/** + * + * @param pBuf + * @return + */ int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf); /** @@ -148,24 +151,45 @@ bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf); /** * Set the buffer page is dirty, and needs to be flushed to disk when swap out. - * @param pPageInfo + * @param pPage * @param dirty */ -void setBufPageDirty(SFilePage* pPageInfo, bool dirty); +void setBufPageDirty(void* pPage, bool dirty); + +/** + * Set the compress/ no-compress flag for paged buffer, when flushing data in disk. + * @param pBuf + */ +void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp); + +/** + * Set the pageId page buffer is not need + * @param pBuf + * @param pageId + */ +void dBufSetBufPageRecycled(SDiskbasedBuf *pBuf, void* pPage); /** * Print the statistics when closing this buffer * @param pBuf */ -void printStatisBeforeClose(SDiskbasedBuf* pBuf); +void dBufSetPrintInfo(SDiskbasedBuf* pBuf); /** - * return buf statistics. + * Return buf statistics. + * @param pBuf + * @return */ SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf); +/** + * Print the buffer statistics information + * @param pBuf + */ +void dBufPrintStatis(const SDiskbasedBuf* pBuf); + #ifdef __cplusplus } #endif -#endif // TDENGINE_TPAGEDBUF_H +#endif // _TD_UTIL_PAGEDBUF_H_ diff --git a/include/util/tqueue.h b/include/util/tqueue.h index cfa5a65c2a14543e6690fcf0b886d77696857cb0..d51184edfca2284f314ab9afe490d72430e8ccdf 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_QUEUE_H -#define _TD_UTIL_QUEUE_H +#ifndef _TD_UTIL_QUEUE_H_ +#define _TD_UTIL_QUEUE_H_ + #include "os.h" #ifdef __cplusplus @@ -47,7 +48,7 @@ typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void * taosAllocateQitem(int32_t size); +void *taosAllocateQitem(int32_t size); void taosFreeQitem(void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); @@ -80,4 +81,4 @@ int32_t taosGetQsetItemsNumber(STaosQset *qset); } #endif -#endif /*_TD_UTIL_QUEUE_H*/ +#endif /*_TD_UTIL_QUEUE_H_*/ diff --git a/include/util/tref.h b/include/util/tref.h index 6680204d63f58a75c61c45c340cc14c9129cd7c0..7e08bb045b08f56afe59436d07576b6e7604c12c 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_REF_H -#define _TD_UTIL_REF_H +#ifndef _TD_UTIL_REF_H_ +#define _TD_UTIL_REF_H_ #include "os.h" @@ -25,45 +25,45 @@ extern "C" { // open a reference set, max is the mod used by hash, fp is the pointer to free resource function // return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately -int taosOpenRef(int max, void (*fp)(void *)); +int32_t taosOpenRef(int32_t max, void (*fp)(void *)); // close the reference set, refId is the return value by taosOpenRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosCloseRef(int refId); +int32_t taosCloseRef(int32_t refId); // add ref, p is the pointer to resource or pointer ID // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately -int64_t taosAddRef(int refId, void *p); +int64_t taosAddRef(int32_t refId, void *p); // remove ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosRemoveRef(int rsetId, int64_t rid); +int32_t taosRemoveRef(int32_t rsetId, int64_t rid); // acquire ref, rid is the reference ID returned by taosAddRef // return the resource p. On error, NULL is returned, and terrno is set appropriately -void *taosAcquireRef(int rsetId, int64_t rid); +void *taosAcquireRef(int32_t rsetId, int64_t rid); // release ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosReleaseRef(int rsetId, int64_t rid); +int32_t taosReleaseRef(int32_t rsetId, int64_t rid); -// return the first reference if rid is 0, otherwise return the next after current reference. +// return the first reference if rid is 0, otherwise return the next after current reference. // if return value is NULL, it means list is over(if terrno is set, it means error happens) -void *taosIterateRef(int rsetId, int64_t rid); +void *taosIterateRef(int32_t rsetId, int64_t rid); // return the number of references in system -int taosListRef(); +int32_t taosListRef(); #define RID_VALID(x) ((x) > 0) -/* sample code to iterate the refs +/* sample code to iterate the refs -void demoIterateRefs(int rsetId) { +void demoIterateRefs(int32_t rsetId) { void *p = taosIterateRef(refId, 0); while (p) { // process P - + // get the rid from p p = taosIterateRef(rsetId, rid); @@ -76,4 +76,4 @@ void demoIterateRefs(int rsetId) { } #endif -#endif /*_TD_UTIL_REF_H*/ +#endif /*_TD_UTIL_REF_H_*/ diff --git a/include/util/tsched.h b/include/util/tsched.h index d60c0c5ad4aa7a7e3e314d53765347b5209c8b84..3bf740f5285bc7559eea0f81f88462db40b28705 100644 --- a/include/util/tsched.h +++ b/include/util/tsched.h @@ -13,8 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_SCHED_H -#define _TD_UTIL_SCHED_H +#ifndef _TD_UTIL_SCHED_H_ +#define _TD_UTIL_SCHED_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -36,7 +38,7 @@ typedef struct SSchedMsg { * @param label the label of the queue * @return the created queue scheduler */ -void *taosInitScheduler(int capacity, int numOfThreads, const char *label); +void *taosInitScheduler(int32_t capacity, int32_t numOfThreads, const char *label); /** * Create a thread-safe ring-buffer based task queue and return the instance. @@ -47,7 +49,7 @@ void *taosInitScheduler(int capacity, int numOfThreads, const char *label); * @param tmrCtrl the timer controller, tmr_ctrl_t* * @return the created queue scheduler */ -void *taosInitSchedulerWithInfo(int capacity, int numOfThreads, const char *label, void *tmrCtrl); +void *taosInitSchedulerWithInfo(int32_t capacity, int32_t numOfThreads, const char *label, void *tmrCtrl); /** * Clean up the queue scheduler instance and free the memory. @@ -68,4 +70,4 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); } #endif -#endif /*_TD_UTIL_SCHED_H*/ +#endif /*_TD_UTIL_SCHED_H_*/ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 823a6f247f865786c0c92d0711863f598ed95abc..64cab08cfe87d430fded815878a5cad6083a4d81 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -16,22 +16,22 @@ #ifndef _TD_UTIL_SKILIST_H #define _TD_UTIL_SKILIST_H -#ifdef __cplusplus -extern "C" { -#endif - #include "os.h" #include "taos.h" #include "tarray.h" #include "tfunctional.h" -#define MAX_SKIP_LIST_LEVEL 15 +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 // For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) #define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 @@ -39,17 +39,17 @@ extern "C" { typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); -typedef void (*sl_patch_row_fn_t)(void * pDst, const void * pSrc); -typedef void* (*iter_next_fn_t)(void *iter); +typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); +typedef void *(*iter_next_fn_t)(void *iter); typedef struct SSkipListNode { - uint8_t level; - void * pData; + uint8_t level; + void *pData; struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_GET_NODE_DATA(n) (n)->pData -#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] /* @@ -100,14 +100,10 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; -typedef enum { - SSkipListPutSuccess = 0, - SSkipListPutEarlyStop = 1, - SSkipListPutSkipOne = 2 -} SSkipListPutStatus; +typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; typedef struct SSkipList { - unsigned int seed; + uint32_t seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; pthread_rwlock_t *lock; @@ -117,41 +113,41 @@ typedef struct SSkipList { uint8_t type; // static info above uint8_t level; uint32_t size; - SSkipListNode * pHead; // point to the first element - SSkipListNode * pTail; // point to the last element + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif - tGenericSavedFunc* insertHandleFn; + tGenericSavedFunc *insertHandleFn; } SSkipList; typedef struct SSkipListIterator { - SSkipList * pSkipList; + SSkipList *pSkipList; SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator - SSkipListNode *next; // next points to the true qualified node in skiplist + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator + SSkipListNode *next; // next points to the true qualified node in skiplist } SSkipListIterator; -#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) -#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) -#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) -#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) -#define SL_SIZE(s) (s)->size +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, __sl_key_fn_t fn); void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); -SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); bool tSkipListIterNext(SSkipListIterator *iter); -SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); -void * tSkipListDestroyIter(SSkipListIterator *iter); +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); @@ -159,4 +155,4 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNod } #endif -#endif /*_TD_UTIL_SKILIST_H*/ +#endif /*_TD_UTIL_SKILIST_H*/ diff --git a/include/util/tstrbuild.h b/include/util/tstrbuild.h index 48a302531cf9c80cc9c8b4ceff3b4ba714127dd1..89f3472346276ca749176c048d9ec118c4b99d19 100644 --- a/include/util/tstrbuild.h +++ b/include/util/tstrbuild.h @@ -13,12 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_STRING_BUILDER_H -#define _TD_UTIL_STRING_BUILDER_H +#ifndef _TD_UTIL_STRING_BUILDER_H_ +#define _TD_UTIL_STRING_BUILDER_H_ -#include -#include -#include +#include "os.h" #ifdef __cplusplus extern "C" { @@ -26,16 +24,16 @@ extern "C" { typedef struct SStringBuilder { jmp_buf jb; - size_t size; - size_t pos; - char* buf; + size_t size; + size_t pos; + char* buf; } SStringBuilder; #define taosStringBuilderSetJmp(sb) setjmp((sb)->jb) -void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size); +void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size); char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len); -void taosStringBuilderDestroy(SStringBuilder* sb); +void taosStringBuilderDestroy(SStringBuilder* sb); void taosStringBuilderAppend(SStringBuilder* sb, const void* data, size_t len); void taosStringBuilderAppendChar(SStringBuilder* sb, char c); @@ -49,4 +47,4 @@ void taosStringBuilderAppendDouble(SStringBuilder* sb, double v); } #endif -#endif /*_TD_UTIL_STRING_BUILDER_H*/ \ No newline at end of file +#endif /*_TD_UTIL_STRING_BUILDER_H_*/ \ No newline at end of file diff --git a/include/util/tthread.h b/include/util/tthread.h index 7a5fd1f4c877024f2e3b8ade79a4094c1ef91ebc..49412069445d0f1cb89631683daa6411ca88879d 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -13,25 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_THREAD_H -#define _TD_UTIL_THREAD_H +#ifndef _TD_UTIL_THREAD_H_ +#define _TD_UTIL_THREAD_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tdef.h" - -// create new thread pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); -// destory thread -bool taosDestoryThread(pthread_t* pthread); -// thread running return true -bool taosThreadRunning(pthread_t* pthread); +bool taosDestoryThread(pthread_t* pthread); +bool taosThreadRunning(pthread_t* pthread); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_THREAD_H*/ +#endif /*_TD_UTIL_THREAD_H_*/ diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 01d70c7d02d65fe4340328297b8057bd6a2731f0..f2ee825c4e5ae4ab8b48184e78e89b5c99cf6ff6 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_TIMER_H -#define _TD_UTIL_TIMER_H +#ifndef _TD_UTIL_TIMER_H_ +#define _TD_UTIL_TIMER_H_ #include "os.h" @@ -25,23 +25,23 @@ extern "C" { typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); -extern int taosTmrThreads; +extern int32_t taosTmrThreads; #define MSECONDS_PER_TICK 5 -void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); +void *taosTmrInit(int32_t maxTmr, int32_t resoultion, int32_t longest, const char *label); -tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void *param, void *handle); +tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *handle); bool taosTmrStop(tmr_h tmrId); bool taosTmrStopA(tmr_h *timerId); -bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void *param, void *handle, tmr_h *pTmrId); +bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *handle, tmr_h *pTmrId); void taosTmrCleanUp(void *handle); -int32_t taosInitTimer(void (*callback)(int), int32_t ms); +int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); void taosUninitTimer(); @@ -49,4 +49,4 @@ void taosUninitTimer(); } #endif -#endif /*_TD_UTIL_TIMER_H*/ +#endif /*_TD_UTIL_TIMER_H_*/ diff --git a/include/util/tutil.h b/include/util/tutil.h index c315948702af31b5dbcf8cde71331f132554532f..e0f92be76a90d5abbfa22366d8513f3e06facbbd 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -13,18 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_UTIL_H -#define _TD_UTIL_UTIL_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_UTIL_UTIL_H_ +#define _TD_UTIL_UTIL_H_ #include "os.h" #include "tcrc32c.h" #include "tdef.h" #include "tmd5.h" +#ifdef __cplusplus +extern "C" { +#endif + int32_t strdequote(char *src); int32_t strndequote(char *dst, const char *z, int32_t len); int32_t strRmquote(char *z, int32_t len); @@ -49,7 +49,7 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, inBuf, (unsigned int)inLen); + tMD5Update(&context, inBuf, (uint32_t)inLen); tMD5Final(&context); memcpy(target, context.digest, tListLen(context.digest)); } @@ -57,17 +57,17 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Update(&context, inBuf, (uint32_t)len); tMD5Final(&context); - sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], - context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], - context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], - context.digest[13], context.digest[14], context.digest[15]); + sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], + context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], + context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_UTIL_H*/ +#endif /*_TD_UTIL_UTIL_H_*/ diff --git a/include/util/tversion.h b/include/util/tversion.h index 3d7a7e1b665f219eee4d9f94a7dd56bc1d0d5bcc..c924752a01f52f01f2cb40dbc6e16770f23d1ea2 100644 --- a/include/util/tversion.h +++ b/include/util/tversion.h @@ -13,8 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_VERSION_H -#define _TD_UTIL_VERSION_H +#ifndef _TD_UTIL_VERSION_H_ +#define _TD_UTIL_VERSION_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -28,4 +30,4 @@ int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t } #endif -#endif /*_TD_UTIL_VERSION_H*/ +#endif /*_TD_UTIL_VERSION_H_*/ diff --git a/include/util/tworker.h b/include/util/tworker.h index 771c7c943359b08e664f3d7de45fe6a4ecf6ada3..e6f6bc077c69f05498df2536c7de55a488dd8f93 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_WORKER_H -#define _TD_UTIL_WORKER_H +#ifndef _TD_UTIL_WORKER_H_ +#define _TD_UTIL_WORKER_H_ + #include "tqueue.h" #ifdef __cplusplus @@ -34,25 +35,25 @@ typedef struct SQWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - STaosQset * qset; - const char * name; - SQWorker * workers; + STaosQset *qset; + const char *name; + SQWorker *workers; pthread_mutex_t mutex; } SQWorkerPool, SFWorkerPool; typedef struct SWWorker { int32_t id; // worker id pthread_t thread; // thread - STaosQall * qall; - STaosQset * qset; // queue set + STaosQall *qall; + STaosQset *qset; // queue set SWWorkerPool *pool; } SWWorker; typedef struct SWWorkerPool { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic - const char * name; - SWWorker * workers; + const char *name; + SWWorker *workers; pthread_mutex_t mutex; } SWWorkerPool; @@ -75,4 +76,4 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); } #endif -#endif /*_TD_UTIL_WORKER_H*/ +#endif /*_TD_UTIL_WORKER_H_*/ diff --git a/include/util/types.h b/include/util/types.h index 25688df56cc0276931419cf24cce5b19ed690d77..f7a535c965fd6deac8dc2eb1d486c072a0e867e6 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -22,14 +22,14 @@ extern "C" { #endif -#define GET_INT8_VAL(x) (*(int8_t *)(x)) -#define GET_INT16_VAL(x) (*(int16_t *)(x)) -#define GET_INT32_VAL(x) (*(int32_t *)(x)) -#define GET_INT64_VAL(x) (*(int64_t *)(x)) -#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) -#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) -#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) -#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t *)(x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) static FORCE_INLINE float taos_align_get_float(const char *pBuf) { #if __STDC_VERSION__ >= 201112L @@ -64,23 +64,33 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { // #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } // #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } // #else - #define GET_FLOAT_VAL(x) (*(float *)(x)) - #define GET_DOUBLE_VAL(x) (*(double *)(x)) - #define SET_BIGINT_VAL(x, y) { (*(int64_t *)(x)) = (int64_t)(y); } - #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } - #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } - #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } - #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) +#define SET_BIGINT_VAL(x, y) \ + { (*(int64_t *)(x)) = (int64_t)(y); } +#define SET_FLOAT_VAL(x, y) \ + { (*(float *)(x)) = (float)(y); } +#define SET_DOUBLE_VAL(x, y) \ + { (*(double *)(x)) = (double)(y); } +#define SET_FLOAT_PTR(x, y) \ + { (*(float *)(x)) = (*(float *)(y)); } +#define SET_DOUBLE_PTR(x, y) \ + { (*(double *)(x)) = (*(double *)(y)); } // #endif -typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) +typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) typedef int32_t VarDataOffsetT; +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + #ifdef __cplusplus } #endif diff --git a/include/util/version.h b/include/util/version.h index 01efecc59d407df3d79c4ea61b2924aace7c3fa2..b241dd248b9a673591bbc94e1a71a78ad721919e 100644 --- a/include/util/version.h +++ b/include/util/version.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_VERSION_H -#define _TD_UTIL_VERSION_H +#ifndef _TD_UTIL_VERSION_H_ +#define _TD_UTIL_VERSION_H_ #ifdef __cplusplus extern "C" { @@ -30,4 +30,4 @@ extern char buildinfo[]; } #endif -#endif /*_TD_UTIL_VERSION_H*/ +#endif /*_TD_UTIL_VERSION_H_*/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index dee80b59e03cf724e93105243ba00627acd96686..8adabe0bac576ae17affb8377c22b2eeb9a61d39 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "parser.h" #include "query.h" #include "taos.h" diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 65887e87dd5c4722eaa56da9beecc21d82fa4a7a..0e60305d40cf3b2d1037bd635d5cb44ab8f50ac6 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -215,23 +215,27 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; - if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { - SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, NULL, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); - if (code != TSDB_CODE_SUCCESS) { - // handle error and retry - } else { - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); - } + SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); + if (code != TSDB_CODE_SUCCESS) { + if (pRequest->body.pQueryJob != NULL) { + schedulerFreeJob(pRequest->body.pQueryJob); } - pRequest->body.resInfo.numOfRows = res.numOfRows; - pRequest->code = res.code; + pRequest->code = code; return pRequest->code; } - return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); + if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { + pRequest->body.resInfo.numOfRows = res.numOfRows; + + if (pRequest->body.pQueryJob != NULL) { + schedulerFreeJob(pRequest->body.pQueryJob); + } + } + + pRequest->code = res.code; + return pRequest->code; } TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index ac36179a264ad8a23c6f7431279b36abe5b68458..7bba0935882bcac5b784dee40a6b4b8f24bc6ae9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,8 +53,10 @@ TEST(testCase, driverInit_Test) { // taos_init(); } -#if 0 +#if 1 TEST(testCase, connect_Test) { +// taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); @@ -332,7 +334,7 @@ TEST(testCase, create_ctable_Test) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + pRes = taos_query(pConn, "create table tu using sts tags('2021-10-10 1:1:1');"); if (taos_errno(pRes) != 0) { printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); } diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c deleted file mode 100644 index ef441c97c786bc51e4db5cfdcac2bc6c5abf5f77..0000000000000000000000000000000000000000 --- a/source/common/src/tcompare.c +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tcompare.h" - - -__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { - __compar_fn_t comparFn = NULL; - - switch (keyType) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - comparFn = (order == TSDB_ORDER_ASC)? compareInt8Val:compareInt8ValDesc; - break; - case TSDB_DATA_TYPE_SMALLINT: - comparFn = (order == TSDB_ORDER_ASC)? compareInt16Val:compareInt16ValDesc; - break; - case TSDB_DATA_TYPE_INT: - comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = (order == TSDB_ORDER_ASC)? compareInt64Val:compareInt64ValDesc; - break; - case TSDB_DATA_TYPE_FLOAT: - comparFn = (order == TSDB_ORDER_ASC)? compareFloatVal:compareFloatValDesc; - break; - case TSDB_DATA_TYPE_DOUBLE: - comparFn = (order == TSDB_ORDER_ASC)? compareDoubleVal:compareDoubleValDesc; - break; - case TSDB_DATA_TYPE_UTINYINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint8Val:compareUint8ValDesc; - break; - case TSDB_DATA_TYPE_USMALLINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint16Val:compareUint16ValDesc; - break; - case TSDB_DATA_TYPE_UINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint32Val:compareUint32ValDesc; - break; - case TSDB_DATA_TYPE_UBIGINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint64Val:compareUint64ValDesc; - break; - case TSDB_DATA_TYPE_BINARY: - comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedStr:compareLenPrefixedStrDesc; - break; - case TSDB_DATA_TYPE_NCHAR: - comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedWStr:compareLenPrefixedWStrDesc; - break; - default: - comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; - break; - } - - return comparFn; -} - -int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { - switch (type) { - case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - case TSDB_DATA_TYPE_UTINYINT: DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); - case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); - case TSDB_DATA_TYPE_UINT: DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); - case TSDB_DATA_TYPE_UBIGINT: DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - case TSDB_DATA_TYPE_NCHAR: { - tstr* t1 = (tstr*) f1; - tstr* t2 = (tstr*) f2; - - if (t1->len != t2->len) { - return t1->len > t2->len? 1:-1; - } - int32_t ret = memcmp((wchar_t*) t1, (wchar_t*) t2, t2->len); - if (ret == 0) { - return ret; - } - return (ret < 0) ? -1 : 1; - } - default: { // todo refactor - tstr* t1 = (tstr*) f1; - tstr* t2 = (tstr*) f2; - - if (t1->len != t2->len) { - return t1->len > t2->len? 1:-1; - } else { - int32_t ret = strncmp(t1->data, t2->data, t1->len); - if (ret == 0) { - return 0; - } else { - return ret < 0? -1:1; - } - } - } - } -} diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0180cc5ede1526748b05986da7001a84b70138ea..f39b20b9347e7f1a441cd51d01c5297ebafef024 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -12,11 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#define _DEFAULT_SOURCE #include "tdataformat.h" -#include "talgo.h" #include "tcoding.h" -#include "wchar.h" -#include "tarray.h" #include "tlog.h" static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -26,7 +25,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i #endif int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; - if(IS_VAR_DATA_TYPE(pCol->type)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { spaceNeeded += sizeof(VarDataOffsetT) * maxPoints; } #ifdef TD_SUPPORT_BITMAP @@ -37,11 +36,10 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += TYPE_BYTES[pCol->type]; #endif - if(pCol->spaceSize < spaceNeeded) { - void* ptr = realloc(pCol->pData, spaceNeeded); - if(ptr == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, - strerror(errno)); + if (pCol->spaceSize < spaceNeeded) { + void *ptr = realloc(pCol->pData, spaceNeeded); + if (ptr == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; } else { pCol->pData = ptr; @@ -67,8 +65,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { * Duplicate the schema and return a new object */ STSchema *tdDupSchema(const STSchema *pSchema) { - - int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); + int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); STSchema *tSchema = (STSchema *)malloc(tlen); if (tSchema == NULL) return NULL; @@ -99,8 +96,8 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { * Decode a schema from a binary. */ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { - int version = 0; - int numOfCols = 0; + int version = 0; + int numOfCols = 0; STSchemaBuilder schemaBuilder; buf = taosDecodeFixedI32(buf, &version); @@ -156,7 +153,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - STColumn* columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); + STColumn *columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); if (columns == NULL) return -1; pBuilder->columns = columns; } @@ -167,7 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { - STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols-1]); + STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]); } @@ -259,7 +256,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); pDataCol->bytes = colBytes(pCol); - pDataCol->offset = colOffset(pCol) + 0; //TD_DATA_ROW_HEAD_SIZE; + pDataCol->offset = colOffset(pCol) + 0; // TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; } @@ -273,7 +270,7 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo return 0; } - if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1; + if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1; if (numOfRows > 0) { // Find the first not null value, fill all previouse values as NULL dataColSetNEleNull(pCol, numOfRows); @@ -304,7 +301,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) } bool isNEleNull(SDataCol *pCol, int nEle) { - if(isAllRowsNull(pCol)) return true; + if (isAllRowsNull(pCol)) return true; for (int i = 0; i < nEle; i++) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } @@ -371,7 +368,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { return NULL; } int i; - for(i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; pCols->cols[i].len = 0; pCols->cols[i].pData = NULL; @@ -387,10 +384,10 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + void *ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for(i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; i++) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; pCols->cols[i].spaceSize = 0; @@ -403,16 +400,16 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { for (i = 0; i < schemaNCols(pSchema); i++) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } - + return 0; } SDataCols *tdFreeDataCols(SDataCols *pCols) { int i; if (pCols) { - if(pCols->cols) { + if (pCols->cols) { int maxCols = pCols->maxCols; - for(i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; i++) { SDataCol *pCol = &pCols->cols[i]; tfree(pCol->pData); } @@ -440,7 +437,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { if (pDataCols->cols[i].len > 0) { - if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { tdFreeDataCols(pRet); return NULL; } @@ -648,9 +645,9 @@ SKVRow tdKVRowDup(SKVRow row) { return trow; } -static int compareColIdx(const void* a, const void* b) { - const SColIdx* x = (const SColIdx*)a; - const SColIdx* y = (const SColIdx*)b; +static int compareColIdx(const void *a, const void *b) { + const SColIdx *x = (const SColIdx *)a; + const SColIdx *y = (const SColIdx *)b; if (x->colId > y->colId) { return 1; } @@ -660,15 +657,13 @@ static int compareColIdx(const void* a, const void* b) { return 0; } -void tdSortKVRowByColIdx(SKVRow row) { - qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); -} +void tdSortKVRowByColIdx(SKVRow row) { qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); } int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SColIdx *pColIdx = NULL; SKVRow row = *orow; SKVRow nrow = NULL; - void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); + void *ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; @@ -700,7 +695,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (IS_VAR_DATA_TYPE(type)) { void *pOldVal = kvRowColVal(row, (SColIdx *)ptr); - if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place + if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); } else { // need to reallocate the memory int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f1faa4f1f439f1f415e0d7059852311..e2880441be2585634a80c1c09d853a8830c73b35 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -1,10 +1,24 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "tep.h" -#include -#include "common.h" +#include "tcompare.h" #include "tglobal.h" -#include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -61,9 +75,8 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } -#define BitmapLen(_n) (((_n) + ((1<> NBIT) -int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { +int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length; @@ -121,14 +134,18 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; switch(type) { + case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} + case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} + case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} default: assert(0); } @@ -249,8 +266,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { } ASSERT(pColInfoData->nullbitmap == NULL); - pDataBlock->info.window.skey = *(TSKEY*) colDataGet(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*) colDataGet(pColInfoData, (pDataBlock->info.rows - 1)); + pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0); + pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); return 0; } @@ -262,8 +279,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); - uint32_t oldLen = colDataGetSize(pCol2, pDest->info.rows); - uint32_t newLen = colDataGetSize(pCol1, pSrc->info.rows); + uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); + uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; char* tmp = realloc(pCol2->pData, newSize); @@ -287,7 +304,7 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - total += colDataGetSize(pColInfoData, pBlock->info.rows); + total += colDataGetLength(pColInfoData, pBlock->info.rows); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { total += sizeof(int32_t) * pBlock->info.rows; @@ -336,7 +353,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (isNull) { // do nothing } else { - char* p = colDataGet(pColInfoData, j); + char* p = colDataGetData(pColInfoData, j); size += varDataTLen(p); } @@ -401,7 +418,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); - char* p = colDataGet(pColData, j); + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } @@ -411,7 +428,6 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return pDst; } - /** * * +------------------+---------------+--------------------+ @@ -444,7 +460,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { pStart += BitmapLen(pBlock->info.rows); } - uint32_t dataSize = colDataGetSize(pCol, numOfRows); + uint32_t dataSize = colDataGetLength(pCol, numOfRows); *(int32_t*) pStart = dataSize; pStart += sizeof(int32_t); @@ -522,6 +538,22 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); } +SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { + SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + pSchema[i].bytes = pColInfoData->info.bytes; + pSchema[i].type = pColInfoData->info.type; + pSchema[i].colId = pColInfoData->info.colId; + } + + if (numOfCols != NULL) { + *numOfCols = pBlock->info.numOfCols; + } + + return pSchema; +} + double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; @@ -577,8 +609,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { } } - void* left1 = colDataGet(pColInfoData, left); - void* right1 = colDataGet(pColInfoData, right); + void* left1 = colDataGetData(pColInfoData, left); + void* right1 = colDataGetData(pColInfoData, right); switch(pColInfoData->info.type) { case TSDB_DATA_TYPE_INT: { @@ -617,7 +649,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return code; } } else { - char* p = colDataGet(pSrc, tupleIndex); + char* p = colDataGetData(pSrc, tupleIndex); code = colDataAppend(pDst, numOfRows, p, false); if (code != TSDB_CODE_SUCCESS) { return code; @@ -956,8 +988,8 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { // } // } -// void* left1 = colDataGet(pColInfoData, left); -// void* right1 = colDataGet(pColInfoData, right); +// void* left1 = colDataGetData(pColInfoData, left); +// void* right1 = colDataGetData(pColInfoData, right); // switch(pColInfoData->info.type) { // case TSDB_DATA_TYPE_INT: { @@ -1041,36 +1073,47 @@ void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { } } -int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(p->info.type)) { - char* tmp = realloc(p->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { + char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - p->varmeta.offset = (int32_t*)tmp; - memset(p->varmeta.offset, 0, sizeof(int32_t) * numOfRows); + pColumn->varmeta.offset = (int32_t*)tmp; + memset(pColumn->varmeta.offset, 0, sizeof(int32_t) * numOfRows); - p->varmeta.length = 0; - p->varmeta.allocLen = 0; - tfree(p->pData); - } else { - char* tmp = realloc(p->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + pColumn->varmeta.length = 0; + pColumn->varmeta.allocLen = 0; + tfree(pColumn->pData); + } else { + char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - p->nullbitmap = tmp; - memset(p->nullbitmap, 0, BitmapLen(numOfRows)); + pColumn->nullbitmap = tmp; + memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); - tmp = realloc(p->pData, numOfRows * p->info.bytes); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumn->pData = tmp; + } + + return TSDB_CODE_SUCCESS; +} - p->pData = tmp; +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + code = blockDataEnsureColumnCapacity(p, numOfRows); + if (code) { + return code; } } @@ -1098,4 +1141,25 @@ void* blockDataDestroy(SSDataBlock* pBlock) { tfree(pBlock->pBlockAgg); tfree(pBlock); return NULL; -} \ No newline at end of file +} + +SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { + int32_t numOfCols = pDataBlock->info.numOfCols; + + SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + colInfo.info = p->info; + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + return pBlock; +} + +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { + return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); +} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 26d33199c1570121ce166459dcb2b0a6d81153eb..db84f87b66c77fd636eb6df1d04efe6c82d108f8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,16 +14,11 @@ */ #define _DEFAULT_SOURCE -#include "os.h" - -#include "taosdef.h" -#include "taoserror.h" +#include "tglobal.h" #include "tcompare.h" #include "tconfig.h" #include "tep.h" -#include "tglobal.h" #include "tlog.h" -#include "tutil.h" SConfig *tsCfg = NULL; @@ -207,6 +202,8 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e } static int32_t taosAddClientLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; @@ -218,8 +215,6 @@ static int32_t taosAddClientLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; - if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; - if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; return 0; } @@ -325,6 +320,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { static void taosSetClientLogCfg(SConfig *pCfg) { SConfigItem *pItem = cfgGetItem(pCfg, "logDir"); tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; @@ -349,12 +345,24 @@ static void taosSetServerLogCfg(SConfig *pCfg) { } static void taosSetClientCfg(SConfig *pCfg) { - tstrncpy(tsFirst, cfgGetItem(pCfg, "firstEp")->str, TSDB_EP_LEN); - tstrncpy(tsSecond, cfgGetItem(pCfg, "secondEp")->str, TSDB_EP_LEN); tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + + SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); + SEp secondEp = {0}; + taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); + cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; @@ -395,6 +403,8 @@ static void taosSetSystemCfg(SConfig *pCfg) { static void taosSetServerCfg(SConfig *pCfg) { tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + taosExpandDir(tsDataDir, tsDataDir, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; @@ -473,10 +483,10 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU if (taosAddClientLogCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1; } else { - if (taosAddClientLogCfg(tsCfg) != 0) return -1; - if (taosAddServerLogCfg(tsCfg) != 0) return -1; if (taosAddClientCfg(tsCfg) != 0) return -1; if (taosAddServerCfg(tsCfg) != 0) return -1; + if (taosAddClientLogCfg(tsCfg) != 0) return -1; + if (taosAddServerLogCfg(tsCfg) != 0) return -1; } taosAddSystemCfg(tsCfg); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8101c18ebf43ce78fb119c77d8eeca6aea6fbd6e..8a3cd0a7183b935da6d81f6f6651c954942f4a3f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tmsg.h" #undef TD_MSG_NUMBER_ @@ -1523,6 +1524,10 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + if (pRsp->vgNum <= 0) { + return 0; + } + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); if (pRsp->pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/common/src/tmsgtype.c b/source/common/src/tmsgtype.c index 6e845e7e9e4f1a7c4b0474e51e11c6e6e3d8f6c9..3ca0f00fc20df54226eb026e2c92e19144de47b2 100644 --- a/source/common/src/tmsgtype.c +++ b/source/common/src/tmsgtype.c @@ -13,6 +13,6 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #define TSDB_SQL_C - #include "tmsgtype.h" diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f6892b26bdb0fc167548d1d95287e5f21bbf6678..e061862856702c8015470afb496bd5e6734e49b6 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -1,9 +1,20 @@ -#include -#include "os.h" -#include "tutil.h" - +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "tname.h" -#include "tmsg.h" #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 91d91f0714d475d427dd7087b27d64fb3f60faf6..861b4dc09334beb2d62e7e0677519c448855a38f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "trow.h" -#include "tarray.h" const uint8_t tdVTypeByte[3] = { TD_VTYPE_NORM_BYTE, // TD_VTYPE_NORM diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index f9755a43b9c4a53473637110bd683556d6c64b10..460c4a6fc08ea07b77a9feec970a87be501e7fd7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -13,20 +13,15 @@ * along with this program. If not, see . */ -#define _BSD_SOURCE - #ifdef DARWIN #define _XOPEN_SOURCE #else #define _XOPEN_SOURCE 500 #endif +#define _BSD_SOURCE #define _DEFAULT_SOURCE - -#include "os.h" -#include "taosdef.h" #include "ttime.h" -#include "tutil.h" /* * mktime64 - Converts date to seconds. @@ -48,50 +43,46 @@ * An encoding of midnight at the end of the day as 24:00:00 - ie. midnight * tomorrow - (allowable under ISO 8601) is supported. */ -static int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, - const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec, int64_t time_zone) -{ - unsigned int mon = mon0, year = year0; +static int64_t user_mktime64(const uint32_t year0, const uint32_t mon0, const uint32_t day, const uint32_t hour, + const uint32_t min, const uint32_t sec, int64_t time_zone) { + uint32_t mon = mon0, year = year0; /* 1..12 -> 11,12,1..10 */ - if (0 >= (int) (mon -= 2)) { - mon += 12; /* Puts Feb last since it has leap day */ + if (0 >= (int32_t)(mon -= 2)) { + mon += 12; /* Puts Feb last since it has leap day */ year -= 1; } - //int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + - // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); + // int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + + // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); int64_t res; - res = 367*((int64_t)mon)/12; - res += year/4 - year/100 + year/400 + day + ((int64_t)year)*365 - 719499; - res = res*24; - res = ((res + hour) * 60 + min) * 60 + sec; + res = 367 * ((int64_t)mon) / 12; + res += year / 4 - year / 100 + year / 400 + day + ((int64_t)year) * 365 - 719499; + res = res * 24; + res = ((res + hour) * 60 + min) * 60 + sec; return (res + time_zone); } // ==== mktime() kernel code =================// static int64_t m_deltaUtc = 0; -void deltaToUtcInitOnce() { +void deltaToUtcInitOnce() { struct tm tm = {0}; - (void)strptime("1970-01-01 00:00:00", (const char *)("%Y-%m-%d %H:%M:%S"), &tm); + (void)strptime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm); m_deltaUtc = (int64_t)mktime(&tm); - //printf("====delta:%lld\n\n", seconds); + // printf("====delta:%lld\n\n", seconds); } static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec); -static char* forwardToTimeStringEnd(char* str); -static bool checkTzPresent(const char *str, int32_t len); +static char* forwardToTimeStringEnd(char* str); +static bool checkTzPresent(const char* str, int32_t len); -static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = { - parseLocaltime, - parseLocaltimeDst -}; +static int32_t (*parseLocaltimeFp[])(char* timestr, int64_t* time, int32_t timePrec) = {parseLocaltime, + parseLocaltimeDst}; int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ @@ -108,9 +99,9 @@ bool checkTzPresent(const char* str, int32_t len) { char* seg = forwardToTimeStringEnd((char*)str); int32_t seg_len = len - (int32_t)(seg - str); - char *c = &seg[seg_len - 1]; - for (int i = 0; i < seg_len; ++i) { - if (*c == 'Z' || *c == 'z' || *c == '+' || *c == '-') { + char* c = &seg[seg_len - 1]; + for (int32_t i = 0; i < seg_len; ++i) { + if (*c == 'Z' || *c == 'z' || *c == '+' || *c == '-') { return true; } c--; @@ -204,13 +195,12 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { i += 2; } - //return error if there're illegal charaters after min(2 Digits) - char *minStr = &str[i]; + // return error if there're illegal charaters after min(2 Digits) + char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { - return -1; + return -1; } - int64_t minute = strnatoi(&str[i], 2); if (minute > 59) { return -1; @@ -238,9 +228,8 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01.123+0800 */ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { - - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); int64_t tzOffset = 0; struct tm tm = {0}; @@ -260,8 +249,8 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); - //int64_t seconds = gmtime(&tm); + int64_t seconds = user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); + // int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); #endif @@ -325,8 +314,9 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { #endif #endif - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); - + int64_t seconds = + user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); + int64_t fraction = 0; if (*str == '.') { @@ -336,8 +326,8 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { } } - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); *time = factor * seconds + fraction; return 0; @@ -355,7 +345,7 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ int64_t seconds = mktime(&tm); - + int64_t fraction = 0; if (*str == '.') { @@ -365,27 +355,22 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { } } - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); *time = factor * seconds + fraction; return 0; } int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || - fromPrecision == TSDB_TIME_PRECISION_MICRO || + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - assert(toPrecision == TSDB_TIME_PRECISION_MILLI || - toPrecision == TSDB_TIME_PRECISION_MICRO || + assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3][3] = { {1., 1000., 1000000.}, - {1.0 / 1000, 1., 1000.}, - {1.0 / 1000000, 1.0 / 1000, 1.} }; + static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { - switch (unit) { case 's': (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); @@ -432,7 +417,8 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time * d - Days (24 hours) * w - Weeks (7 days) */ -int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, + int32_t timePrecision) { errno = 0; char* endPtr = NULL; @@ -479,9 +465,9 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { } struct tm tm; - time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); + time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); localtime_r(&tt, &tm); - int mon = tm.tm_year * 12 + tm.tm_mon + (int)duration; + int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)duration; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -502,13 +488,13 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; - time_t t = (time_t)skey; + time_t t = (time_t)skey; localtime_r(&t, &tm); - int smon = tm.tm_year * 12 + tm.tm_mon; + int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; localtime_r(&t, &tm); - int emon = tm.tm_year * 12 + tm.tm_mon; + int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { interval *= 12; @@ -527,7 +513,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; - time_t tt = (time_t)start; + time_t tt = (time_t)start; localtime_r(&tt, &tm); tm.tm_sec = 0; tm.tm_min = 0; @@ -536,10 +522,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (pInterval->slidingUnit == 'y') { tm.tm_mon = 0; - tm.tm_year = (int)(tm.tm_year / pInterval->sliding * pInterval->sliding); + tm.tm_year = (int32_t)(tm.tm_year / pInterval->sliding * pInterval->sliding); } else { - int mon = tm.tm_year * 12 + tm.tm_mon; - mon = (int)(mon / pInterval->sliding * pInterval->sliding); + int32_t mon = tm.tm_year * 12 + tm.tm_mon; + mon = (int32_t)(mon / pInterval->sliding * pInterval->sliding); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; } @@ -552,17 +538,17 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (delta / pInterval->sliding + factor) * pInterval->sliding; if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ // todo refactor to extract function that is available for Linux/Windows/Mac platform - #if defined(WINDOWS) && _MSC_VER >= 1900 +#if defined(WINDOWS) && _MSC_VER >= 1900 // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 int64_t timezone = _timezone; int32_t daylight = _daylight; char** tzname = _tzname; - #endif +#endif start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); } @@ -573,7 +559,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = start + pInterval->interval - 1; - while(end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window start += pInterval->sliding; if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { @@ -606,8 +592,8 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // and the parameter can also be a variable. const char* fmtts(int64_t ts) { static char buf[96]; - size_t pos = 0; - struct tm tm; + size_t pos = 0; + struct tm tm; if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; @@ -624,7 +610,7 @@ const char* fmtts(int64_t ts) { buf[pos++] = ' '; } pos += strftime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm); - pos += sprintf(buf + pos, ".%03d", (int)(ts % 1000)); + pos += sprintf(buf + pos, ".%03d", (int32_t)(ts % 1000)); } { @@ -636,7 +622,7 @@ const char* fmtts(int64_t ts) { buf[pos++] = ' '; } pos += strftime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm); - pos += sprintf(buf + pos, ".%06d", (int)(ts % 1000000)); + pos += sprintf(buf + pos, ".%06d", (int32_t)(ts % 1000000)); } return buf; diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 218472e7c91af013153ffd500b200258accfd905..f20e91168162757f98529de72f5e293843df0c3c 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -1,11 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "ttszip.h" -#include #include "taoserror.h" #include "tcompression.h" -#include "tutil.h" static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); @@ -22,7 +36,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } pTSBuf->autoDelete = autoDelete; - + taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); @@ -34,20 +48,20 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { if (!autoDelete) { remove(pTSBuf->path); } - + if (NULL == allocResForTSBuf(pTSBuf)) { return NULL; } - + // update the header info STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); - + tsBufResetPos(pTSBuf); pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->tsOrder = order; - + return pTSBuf; } @@ -58,23 +72,23 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } pTSBuf->autoDelete = autoDelete; - + tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } - + if (allocResForTSBuf(pTSBuf) == NULL) { return NULL; } - + // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); @@ -84,7 +98,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufDestroy(pTSBuf); return NULL; } - + if (header.numOfGroup > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfGroup; STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); @@ -92,57 +106,58 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->pData = tmp; } - + pTSBuf->numOfGroups = header.numOfGroup; - + // check the ts order pTSBuf->tsOrder = header.tsOrder; if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { -// tscError("invalid order info in buf:%d", pTSBuf->tsOrder); + // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); tsBufDestroy(pTSBuf); return NULL; } - + size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); - return NULL; - } - - //int64_t pos = ftell(pTSBuf->pFile); //pos not used + return NULL; + } + + // int64_t pos = ftell(pTSBuf->pFile); //pos not used sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); - + // the length value for each vnode is not kept in file, so does not set the length value for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } free(buf); - + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - + int64_t file_size; if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); - + // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; - -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->pFile), -// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); - + + // tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, + // fileno(pTSBuf->pFile), + // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); + return pTSBuf; } @@ -150,22 +165,22 @@ void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return NULL; } - + tfree(pTSBuf->assistBuf); tfree(pTSBuf->tsData.rawBuf); - + tfree(pTSBuf->pData); tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { taosCloseFile(&pTSBuf->pFile); } - + if (pTSBuf->autoDelete) { -// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); + // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); remove(pTSBuf->path); } else { -// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); + // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } taosVariantDestroy(&pTSBuf->block.tag); @@ -175,7 +190,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { int32_t last = pTSBuf->numOfGroups - 1; - + assert(last >= 0); return &pTSBuf->pData[last]; } @@ -184,40 +199,40 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } - + pTSBuf->pData = tmp; pTSBuf->numOfAlloc = newSize; memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); } - + if (pTSBuf->numOfGroups > 0) { STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); } - + // set initial value for vnode block STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; assert(pBlockInfo->offset >= getDataStartOffset()); - + // update vnode info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); - + // add one vnode info pTSBuf->numOfGroups += 1; - + // update the header info STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - + STSBufUpdateHeader(pTSBuf, &header); return tsBufGetLastGroupInfo(pTSBuf); } @@ -226,7 +241,7 @@ static void shrinkBuffer(STSList* ptsData) { // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size if (ptsData->allocSize >= ptsData->threshold * 2) { char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); - if(rawBuf) { + if (rawBuf) { ptsData->rawBuf = rawBuf; ptsData->allocSize = MEM_BUF_SIZE; } @@ -246,18 +261,17 @@ static void writeDataToDisk(STSBuf* pTSBuf) { if (pTSBuf->tsData.len == 0) { return; } - + STSBlock* pBlock = &pTSBuf->block; STSList* pTsData = &pTSBuf->tsData; pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; - pBlock->compLen = - tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, - TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - + pBlock->compLen = tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len / TSDB_KEYSIZE, pBlock->payload, + pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); - + /* * format for output data: * 1. tags, number of ts, size after compressed, payload, size after compressed @@ -275,10 +289,10 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); } else { trueLen = 0; metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); @@ -289,19 +303,19 @@ static void writeDataToDisk(STSBuf* pTSBuf) { taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; - + pTSBuf->tsData.len = 0; - + STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + pGroupBlockInfoEx->info.compLen += blockSize; pGroupBlockInfoEx->info.numOfBlocks += 1; - + shrinkBuffer(&pTSBuf->tsData); } @@ -312,7 +326,7 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) { if (tmp == NULL) { // todo } - + ptsData->rawBuf = tmp; ptsData->allocSize = newSize; } @@ -322,8 +336,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { STSBlock* pBlock = &pTSBuf->block; // clear the memory buffer - pBlock->compLen = 0; - pBlock->padding = 0; + pBlock->compLen = 0; + pBlock->padding = 0; pBlock->numOfElem = 0; int32_t offset = -1; @@ -333,11 +347,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block */ - int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); + int32_t prev = -(int32_t)(sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); - size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); - UNUSED(sz); + UNUSED(sz); pBlock->compLen = pBlock->padding; @@ -362,11 +376,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t) pBlock->tag.nLen); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t) pBlock->tag.nLen); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { // TODO check the return value + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); UNUSED(sz); } @@ -381,7 +395,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); } - + // read the comp length at the length of comp block sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); @@ -395,24 +409,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } UNUSED(sz); - + // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } - + return pBlock; } // set the order of ts buffer if the ts order has not been set yet static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { STSList* ptsData = &pTSBuf->tsData; - + if (pTSBuf->tsOrder == -1) { if (ptsData->len > 0) { TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); - + if (lastKey > *(TSKEY*)pData) { pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { @@ -422,7 +436,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { // no data in current vnode, more than one ts is added, check the orders TSKEY k1 = *(TSKEY*)(pData); TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); - + if (k1 < k2) { pTSBuf->tsOrder = TSDB_ORDER_ASC; } else if (k1 > k2) { @@ -434,23 +448,23 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { } else { // todo the timestamp order is set, check the asc/desc order of appended data } - + return TSDB_CODE_SUCCESS; } void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len) { STSGroupBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; - + if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); - + pBlockInfo = addOneGroupInfo(pTSBuf, id); } else { pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - + assert(pBlockInfo->info.id == id); if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { @@ -462,22 +476,22 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, i taosVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); - + // todo check return value setCheckTSOrder(pTSBuf, pData, len); - + ptsData->len += len; pBlockInfo->len += len; - + pTSBuf->numOfTotal += len / TSDB_KEYSIZE; - + // the size of raw data exceeds the size of the default prepared buffer, so // during getBufBlock, the output buffer needs to be large enough. if (ptsData->len >= ptsData->threshold) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); } - + tsBufResetPos(pTSBuf); } @@ -485,15 +499,15 @@ void tsBufFlush(STSBuf* pTSBuf) { if (pTSBuf->tsData.len <= 0) { return; } - + writeDataToDisk(pTSBuf); shrinkBuffer(&pTSBuf->tsData); - + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); - + // save the ts order into header STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; @@ -508,7 +522,7 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num break; } } - + return j; } @@ -517,17 +531,17 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } - + // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode int32_t i = 0; bool decomp = false; - + while ((i++) <= blockIndex) { if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { return -1; } } - + // set the file position to be the end of previous comp block if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; @@ -536,34 +550,34 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } - + return 0; } static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, SVariant* tag) { bool decomp = false; - + int64_t offset = 0; if (pTSBuf->cur.order == TSDB_ORDER_ASC) { offset = pBlockInfo->offset; } else { // reversed traverse starts from the end of block offset = pBlockInfo->offset + pBlockInfo->compLen; } - + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - + for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { return -1; } - + if (taosVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); + return (pTSBuf->cur.order == TSDB_ORDER_ASC) ? i : (pBlockInfo->numOfBlocks - (i + 1)); } } - + return -1; } @@ -572,14 +586,14 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex if (pBlockInfo->numOfBlocks <= blockIndex) { assert(false); } - + STSCursor* pCur = &pTSBuf->cur; if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { + (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; int32_t step = abs(blockIndex - pCur->blockIndex); - + while ((++i) <= step) { if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { return; @@ -590,11 +604,11 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex assert(false); } } - + STSBlock* pBlock = &pTSBuf->block; - + size_t s = pBlock->numOfElem * TSDB_KEYSIZE; - + /* * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function @@ -602,16 +616,16 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex if (s > pTSBuf->tsData.allocSize) { expandBuffer(&pTSBuf->tsData, (int32_t)s); } - + pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - + assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - + pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; - + pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } @@ -633,7 +647,7 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { if (j == -1) { return NULL; } - + return &pTSBuf->pData[j].info; } @@ -646,13 +660,14 @@ int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { -// qError("fseek failed, errno:%d", errno); + // qError("fseek failed, errno:%d", errno); return -1; } size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); - if (ws != 1) { -// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); + if (ws != 1) { + // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, + // (int32_t)sizeof(STSBufFileHeader)); return -1; } return 0; @@ -662,33 +677,33 @@ bool tsBufNextPos(STSBuf* pTSBuf) { if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { return false; } - + STSCursor* pCur = &pTSBuf->cur; - + // get the first/last position according to traverse order if (pCur->vgroupIndex == -1) { if (pCur->order == TSDB_ORDER_ASC) { tsBufGetBlock(pTSBuf, 0, 0); - + if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return tsBufResetPos(pTSBuf); return false; } else { return true; } - + } else { // get the last timestamp record in the last block of the last vnode assert(pTSBuf->numOfGroups > 0); - + int32_t groupIndex = pTSBuf->numOfGroups - 1; pCur->vgroupIndex = groupIndex; - + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - + tsBufGetBlock(pTSBuf, groupIndex, blockIndex); - + pCur->tsIndex = pTSBuf->block.numOfElem - 1; if (pTSBuf->block.numOfElem == 0) { tsBufResetPos(pTSBuf); @@ -698,16 +713,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } } - + int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; - + while (1) { assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - + if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { @@ -716,15 +731,15 @@ bool tsBufNextPos(STSBuf* pTSBuf) { pCur->vgroupIndex = -1; return false; } - + if (pBlockInfo == NULL) { return false; } - + int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; - + } else { tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); break; @@ -734,7 +749,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { break; } } - + return true; } @@ -742,7 +757,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return; } - + pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; } @@ -751,14 +766,14 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return elem1; } - + STSCursor* pCur = &pTSBuf->cur; if (pCur != NULL && pCur->vgroupIndex < 0) { return elem1; } STSBlock* pBlock = &pTSBuf->block; - + elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); elem1.tag = &pBlock->tag; @@ -777,65 +792,65 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { return 0; } - + if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { return -1; } - + // src can only have one vnode index assert(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); - + // compared with the last vnode id - int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; + int32_t id = tsBufGetLastGroupInfo((STSBuf*)pSrcBuf)->info.id; if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { int32_t oldSize = pDestBuf->numOfGroups; int32_t newSize = oldSize + pSrcBuf->numOfGroups; - + if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - + STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } - + pDestBuf->pData = tmp; } - + // directly copy the vnode index information memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); - + // set the new offset value for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; pBlockInfoEx->info.id = id; } - + pDestBuf->numOfGroups = newSize; } else { STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); - + pBlockInfoEx->len += pSrcBuf->pData[0].len; pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; pBlockInfoEx->info.id = id; } - + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); - + int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); - + if (written == -1 || written != size) { return -1; } - + pDestBuf->numOfTotal += pSrcBuf->numOfTotal; int32_t oldSize = pDestBuf->fileSize; @@ -850,7 +865,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { int64_t file_size; if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { - return -1; + return -1; } pDestBuf->fileSize = (uint32_t)file_size; @@ -861,33 +876,33 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { STSBuf* pTSBuf = tsBufCreate(true, order); - + STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); pBlockInfo->numOfBlocks = numOfBlocks; pBlockInfo->compLen = len; pBlockInfo->offset = getDataStartOffset(); pBlockInfo->id = id; - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { -// qError("fseek failed, errno:%d", errno); + // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { -// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); + // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); return NULL; } pTSBuf->fileSize += len; - + pTSBuf->tsOrder = order; assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - + STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; if (STSBufUpdateHeader(pTSBuf, &header) < 0) { @@ -896,42 +911,42 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->pFile)) == -1) { -//// qError("fsync failed, errno:%d", errno); -// tsBufDestroy(pTSBuf); -// return NULL; -// } - + // if (taosFsync(fileno(pTSBuf->pFile)) == -1) { + //// qError("fsync failed, errno:%d", errno); + // tsBufDestroy(pTSBuf); + // return NULL; + // } + return pTSBuf; } STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag) { STSElem elem = {.id = -1}; - + if (pTSBuf == NULL) { return elem; } - + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return elem; } - + // for debug purpose // tsBufDisplay(pTSBuf); - + STSCursor* pCur = &pTSBuf->cur; STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; - + int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); if (blockIndex < 0) { return elem; } - + pCur->vgroupIndex = j; pCur->blockIndex = blockIndex; tsBufGetBlock(pTSBuf, j, blockIndex); - + return tsBufGetElem(pTSBuf); } @@ -940,7 +955,7 @@ STSCursor tsBufGetCursor(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return c; } - + return pTSBuf->cur; } @@ -948,12 +963,12 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { if (pTSBuf == NULL || pCur == NULL) { return; } - + // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); if (pCur->vgroupIndex != -1) { tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); } - + pTSBuf->cur = *pCur; } @@ -961,7 +976,7 @@ void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { if (pTSBuf == NULL) { return; } - + pTSBuf->cur.order = order; } @@ -978,19 +993,19 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) { void tsBufDisplay(STSBuf* pTSBuf) { printf("-------start of ts comp file-------\n"); printf("number of vnode:%d\n", pTSBuf->numOfGroups); - + int32_t old = pTSBuf->cur.order; pTSBuf->cur.order = TSDB_ORDER_ASC; - + tsBufResetPos(pTSBuf); - + while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i, elem.ts); } } - + pTSBuf->cur.order = old; printf("-------end of ts comp file-------\n"); } @@ -1007,36 +1022,36 @@ static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { const int32_t INITIAL_GROUPINFO_SIZE = 4; - + pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->bufSize = MEM_BUF_SIZE; pTSBuf->tsData.threshold = MEM_BUF_SIZE; pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - + pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->block.payload = malloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->fileSize += getDataStartOffset(); return pTSBuf; } @@ -1062,28 +1077,28 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { (*id)[i] = pTSBuf->pData[i].info.id; } } int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); - STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; *len = 0; *numOfBlocks = 0; if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); -// qError("%p: fseek failed: %s", pSql, tstrerror(code)); + int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); + // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(taosEOFFile(pTSBuf->pFile)); -// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); + // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } @@ -1106,6 +1121,4 @@ STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag) { return el; } -bool tsBufIsValidElem(STSElem* pElem) { - return pElem->id >= 0; -} +bool tsBufIsValidElem(STSElem* pElem) { return pElem->id >= 0; } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index ee32a209203ace3e7b484c5729882d4ab57f8eab..f32698c98ad7179f2661c85fec2df3906aa76dc0 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "ttypes.h" -#include "../../../include/client/taos.h" -#include "os.h" #include "tcompression.h" const int32_t TYPE_BYTES[15] = { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 27f1d4947d999fa5e183b9283e82be719cb3eb01..ceef12e986105ad42b1322db26c20a19a760f740 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -12,14 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "taos.h" -#include "taosdef.h" +#define _DEFAULT_SOURCE +#include "tvariant.h" #include "ttime.h" #include "ttokendef.h" -#include "ttypes.h" -#include "tutil.h" #include "tvariant.h" #define SET_EXT_INFO(converted, res, minv, maxv, exti) \ diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index e9e8d086b3c75cb97c2bbb9a14b0497cc2632b62..4821d60875f9f3daaf7e5a2fa0a66f1bc5cd0af3 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,4 +1,4 @@ -#include +#include "tcommon.h" #include #include #include @@ -162,7 +162,7 @@ TEST(testCase, Datablock_test) { ASSERT_EQ(colDataGetNumOfCols(b), 2); ASSERT_EQ(colDataGetNumOfRows(b), 40); - char* pData = colDataGet(p1, 3); + char* pData = colDataGetData(p1, 3); printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c index 57c788dee19aad9647178179c068cd139939ad15..afb9ffbdd0b43cf22e8fdcacf4eabb8156faf2d3 100644 --- a/source/dnode/mgmt/daemon/src/dmnCfg.c +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -22,11 +22,11 @@ SDnodeObjCfg dmnGetObjCfg() { SDnodeObjCfg objCfg = {0}; objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - tstrncpy(objCfg.dataDir, cfgGetItem(pCfg, "dataDir")->str, sizeof(objCfg.dataDir)); - tstrncpy(objCfg.firstEp, cfgGetItem(pCfg, "firstEp")->str, sizeof(objCfg.firstEp)); - tstrncpy(objCfg.secondEp, cfgGetItem(pCfg, "secondEp")->str, sizeof(objCfg.firstEp)); - objCfg.serverPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; - tstrncpy(objCfg.localFqdn, cfgGetItem(pCfg, "fqdn")->str, sizeof(objCfg.localFqdn)); + tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir)); + tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp)); + objCfg.serverPort = tsServerPort; + tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn)); snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); objCfg.pDisks = tsDiskCfg; objCfg.numOfDisks = tsDiskCfgNum; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 8c64cc16c34f6ed0fbf573d5b2e36787f5070be9..dfa2b747556989bf7cde63bc038e7c1d97444e84 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -95,7 +95,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STB_META)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; diff --git a/include/util/tstep.h b/source/dnode/mnode/impl/inc/mndInfoSchema.h similarity index 55% rename from include/util/tstep.h rename to source/dnode/mnode/impl/inc/mndInfoSchema.h index e4ce7f584b368b87c5e0636abed30d9e201b7f83..7db41254027c3c2924e094ae806bbcc8907f0981 100644 --- a/include/util/tstep.h +++ b/source/dnode/mnode/impl/inc/mndInfoSchema.h @@ -13,25 +13,33 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_STEP_H_ -#define _TD_UTIL_STEP_H_ +#ifndef _TD_MND_INFO_SCHEMA_H_ +#define _TD_MND_INFO_SCHEMA_H_ + +#include "mndInt.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SSteps SSteps; -typedef int32_t (*InitFp)(); -typedef void (*CleanupFp)(); -typedef void (*ReportFp)(char *name, char *desc); +typedef struct SInfosTableSchema { + char *name; + int32_t type; + int32_t bytes; +} SInfosTableSchema; + +typedef struct SInfosTableMeta { + char *name; + const SInfosTableSchema *schema; + int32_t colNum; +} SInfosTableMeta; -SSteps *taosStepInit(int32_t maxsize, ReportFp fp); -int32_t taosStepExec(SSteps *steps); -void taosStepCleanup(SSteps *steps); -int32_t taosStepAdd(SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp); +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); +int32_t mndInitInfos(SMnode *pMnode); +void mndCleanupInfos(SMnode *pMnode); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_STEP_H_*/ +#endif /*_TD_MND_INFO_SCHEMA_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index e65535206d6cc1e23ae86d070b39d9c53336da41..54595fb10533bdd3942688a450a17b0eacf6b6b2 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -90,6 +90,7 @@ typedef struct SMnode { SProfileMgmt profileMgmt; STelemMgmt telemMgmt; SSyncMgmt syncMgmt; + SHashObj *infosMeta; MndMsgFp msgFp[TDMT_MAX]; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c8a4ce586fc81db61759df6819435f1c9911680..9165fa2264725253f9a1082f47ae26ae27a0f7d4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -937,36 +937,41 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { goto USE_DB_OVER; } - pDb = mndAcquireDb(pMnode, usedbReq.db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto USE_DB_OVER; - } + char *p = strchr(usedbReq.db, '.'); + if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { + memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + } else { + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto USE_DB_OVER; - } + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } - if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; - } + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; - } + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } - if (usedbReq.vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - } + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c new file mode 100644 index 0000000000000000000000000000000000000000..2c391e93e8ba0d6d0fd9950cfd3139215a043aed --- /dev/null +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndInfoSchema.h" + +static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vnodes", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cores", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "offline_reason", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, + {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; + +static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, + {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, + {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, + {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, + {TSDB_INS_TABLE_USER_DATABASE, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, + {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, + {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, + {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)}, + {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, + {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, + }; + + +int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { + SSchema *schema = calloc(colNum, sizeof(SSchema)); + if (NULL == schema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + + for (int32_t i = 0; i < colNum; ++i) { + strcpy(schema[i].name, pSrc[i].name); + + schema[i].type = pSrc[i].type; + schema[i].colId = i + 1; + schema[i].bytes = pSrc[i].bytes; + } + + *pDst = schema; + + return TSDB_CODE_SUCCESS; +} + +int32_t mndInsInitMeta(SHashObj *hash) { + STableMetaRsp meta = {0}; + + strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); + meta.tableType = TSDB_NORMAL_TABLE; + meta.sversion = 1; + meta.tversion = 1; + + for (int32_t i = 0; i < tListLen(infosMeta); ++i) { + strcpy(meta.tbName, infosMeta[i].name); + meta.numOfColumns = infosMeta[i].colNum; + + if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) { + return -1; + } + + if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { + if (NULL == pMnode->infosMeta) { + terrno = TSDB_CODE_MND_NOT_READY; + return -1; + } + + STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); + if (NULL == meta) { + mError("invalid information schema table name:%s", tbName); + terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + return -1; + } + + *pRsp = *meta; + + pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pRsp->pSchemas = NULL; + return -1; + } + + memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); + + return 0; +} + +int32_t mndInitInfos(SMnode *pMnode) { + pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pMnode->infosMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return mndInsInitMeta(pMnode->infosMeta); +} + +void mndCleanupInfos(SMnode *pMnode) { + if (NULL == pMnode->infosMeta) { + return; + } + + void *pIter = taosHashIterate(pMnode->infosMeta, NULL); + while (pIter) { + STableMetaRsp *meta = (STableMetaRsp *)pIter; + + tfree(meta->pSchemas); + + pIter = taosHashIterate(pMnode->infosMeta, pIter); + } + + taosHashCleanup(pMnode->infosMeta); + pMnode->infosMeta = NULL; +} + + + diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 274ae100455c1bbd65ff746423103cbde8d8ab63..d8091dfa760bc939d60867852d74aae459aa207a 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" #include "tname.h" #define TSDB_STB_VER_NUMBER 1 @@ -38,7 +39,7 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq); +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -58,7 +59,7 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STB_META, mndProcessStbMetaReq); + mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -1310,7 +1311,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; int32_t code = -1; STableInfoReq infoReq = {0}; @@ -1321,9 +1322,16 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { goto RETRIEVE_META_OVER; } - mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + mDebug("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } + } else { + mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } } int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); @@ -1553,4 +1561,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 7c1215ea422d5026715d35efae90112af24e1ff9..d6a4c76c6220b8b0cc796a48497ea45141c5a4d1 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -63,7 +63,7 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { int32_t done = 0; // FILE* fp = fopen("/proc/cpuinfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return; } @@ -93,7 +93,7 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; // FILE* fp = fopen("/etc/os-release", "r"); - TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return; } @@ -120,7 +120,7 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; // FILE* fp = fopen("/proc/meminfo", "r"); - TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7cdae79c0d35db885cf521721192396d1094dd56..b1cdf484b4805a9355da48db15cf5df5ca7ea91b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -19,7 +19,7 @@ #include "mndDb.h" #include "mndShow.h" #include "mndTrans.h" -#include "tkey.h" +#include "tbase64.h" #define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 676cbfa1b3d03c2a59ce4944675fe67408dd54e0..ab5d0d794b6f0416056f418dbc83d104f7362d69 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -36,6 +36,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" #define MQ_TIMER_MS 3000 #define TRNAS_TIMER_MS 6000 @@ -221,6 +222,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { @@ -503,9 +505,17 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } } + +// Note: uid 0 is reserved uint64_t mndGenerateUid(char *name, int32_t len) { - int64_t us = taosGetTimestampUs(); int32_t hashval = MurmurHash3_32(name, len); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + + do { + int64_t us = taosGetTimestampUs(); + uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; + uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + if (uuid) { + return uuid; + } + } while (true); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index a3efad8aa20e57eb1d4d8910f790f077e7cc4253..17fda48cd7ec3d132a290757e1ef74367316ba3c 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -119,8 +119,8 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { EXPECT_EQ(test.GetShowRows(), 2); CheckInt32(2); CheckInt32(3); - CheckInt32(0); - CheckInt32(0); + IgnoreInt32(); + IgnoreInt32(); CheckInt16(1); CheckInt16(1); CheckBinary("master", 9); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c551097e720e32b5e303bd5180b8a9ad18edb130..f45c0795cd6441546a351a0e814f7a25c5d5ae95 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -339,7 +339,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { void* pReq = rpcMallocCont(contLen); tSerializeSTableInfoReq(pReq, contLen, &infoReq); - SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 44a352ec54966af9a1cb13c767cc8a23988395c6..c98a97d0dc19d669980a0baa2370243b22aebdf3 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -16,8 +16,7 @@ #ifndef _TD_META_H_ #define _TD_META_H_ -#include "mallocator.h" -#include "os.h" +#include "tmallocator.h" #include "tmsg.h" #include "trow.h" diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index a516f423bbeb6c9df5746da932977d299c6ec856..ffab15f39446617cb33ab7d878a1a9e3fdca1f26 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -16,11 +16,10 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ -#include "common.h" +#include "tcommon.h" #include "executor.h" -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" -#include "os.h" #include "scheduler.h" #include "taoserror.h" #include "tlist.h" diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 7bdd8fc266ac9900bb6e719f65de87beb2e68a06..677ca9c3367a85045318bd1d09f75d0fa725f188 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -16,9 +16,9 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" -#include "common.h" +#include "tcommon.h" #include "tfs.h" #ifdef __cplusplus diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 3a06674e3ce50a58a23800f50d55ca76bed46046..31f04e840ac2a82fc95dac80461cd5efb5c07d27 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -216,13 +216,15 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); - // pHandle->tbUid = tbUid; } + return 0; } diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index b76c08b7b4a9a59081e3de6eac70b8e1aedafe15..f4128f7170675633664f1cb6f6685563eaccf70c 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -16,7 +16,7 @@ #ifndef _TD_META_DEF_H_ #define _TD_META_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 55e16218ad71fffc89306489d9b2d260ce93f4f7..98c0de32a851fdfc1d50366f77dfaf8d16303f61 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -16,7 +16,7 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" #include "tcompression.h" #include "tglobal.h" diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 97cd4e2612c7a9202fa33f35e19e917bb6a4b719..8ecb4e4a6ebe7a8b15a7eeef8cb6d48e5eb8d889 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -22,7 +22,7 @@ #include "tsdbFile.h" #include "tskiplist.h" #include "tsdbMemory.h" -#include "common.h" +#include "tcommon.h" typedef struct SReadH SReadH; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index fb81ddbc5ce992c699faa4cecf1d0d16a429fbcd..75088687a810ea461daaa04793005518c0e659b7 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -16,7 +16,7 @@ #ifndef _TD_VNODE_DEF_H_ #define _TD_VNODE_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" // #include "sync.h" #include "tcoding.h" #include "tfs.h" diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bed2c0cd41d3b2e37d781c98d9853c59a2d89771..c1cf776cb542110f6089ab9e684d2e2756a8040c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -19,7 +19,7 @@ #include "tsdbLog.h" #include "tsdbReadImpl.h" #include "ttime.h" -#include "exception.h" +#include "texception.h" #include "os.h" #include "talgo.h" #include "tcompare.h" @@ -88,7 +88,7 @@ typedef struct STableCheckInfo { int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks uint8_t chosen:2; // indicate which iterator should move forward - bool initBuf; // whether to initialize the in-memory skip list iterator or not + bool initBuf:1; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator } STableCheckInfo; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5857437f9d6381e34620c093c8532cdb847ffb6b..e71f559ad960466264702e67b6c384d01e5f1113 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "catalog.h" -#include "common.h" +#include "tcommon.h" #include "query.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 @@ -189,11 +189,21 @@ typedef struct SCtgAction { #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) -#define CTG_IS_STABLE(isSTable) (1 == (isSTable)) -#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable)) -#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0) -#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) -#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_FLAG_STB 0x1 +#define CTG_FLAG_NOT_STB 0x2 +#define CTG_FLAG_UNKNOWN_STB 0x4 +#define CTG_FLAG_INF_DB 0x8 + +#define CTG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) +#define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) +#define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) +#define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) +#define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) +#define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) +#define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) + +#define CTG_IS_INF_DBNAME(_dbname) ((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d62c189d33cd7c7621f8c65d3cf8981b9cf754b2..2b6caa2f8fb91f791f2ab30989398e9e12bf754e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -509,7 +509,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); @@ -517,7 +517,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } *pTableMeta = NULL; @@ -590,15 +594,19 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType, int32_t flag) { if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } + SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -629,7 +637,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -637,21 +645,21 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { ctgError("Build mnode stablemeta msg failed, code:%x", code); CTG_ERR_RET(code); } SRpcMsg rpcMsg = { - .msgType = TDMT_MND_STB_META, + .msgType = TDMT_MND_TABLE_META, .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -664,7 +672,7 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S CTG_ERR_RET(rpcRsp.code); } - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); @@ -675,15 +683,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -709,7 +717,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSe }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -829,7 +837,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { +int32_t ctgStbVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { @@ -1078,6 +1086,10 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + if (CTG_IS_INF_DBNAME(dbFName)) { + return TSDB_CODE_SUCCESS; + } + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); @@ -1100,7 +1112,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -1258,7 +1270,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); } } @@ -1428,15 +1440,17 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + if (!CTG_IS_INF_DB(flag)) { + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); + } SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; @@ -1445,33 +1459,37 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - - if (CTG_IS_STABLE(isSTable)) { + + if (CTG_IS_INF_DB(flag)) { + ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); + + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); + } else if (CTG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); if (CTG_IS_META_NULL(output->metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); tfree(output->tbMeta); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); @@ -1530,7 +1548,7 @@ _return: CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1538,26 +1556,30 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t exist = 0; int32_t code = 0; - if (!forceUpdate) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + CTG_SET_INF_DB(flag); + } - if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { + if ((!forceUpdate) || (CTG_IS_INF_DB(flag))) { + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag)); + + if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } tfree(*pTableMeta); - } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { + } else if (CTG_IS_UNKNOWN_STB(flag)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType, flag)); - CTG_SET_STABLE(isSTable, tbType); + CTG_SET_STB(flag, tbType); } STableMetaOutput *output = NULL; while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; @@ -1582,7 +1604,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag)); if (0 == exist) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; @@ -1663,6 +1685,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); @@ -1729,7 +1756,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionCompare)); ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2140,16 +2167,16 @@ _return: } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_STB)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2204,20 +2231,20 @@ _return: } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_GEN_STB_FLAG(isSTable), NULL)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, true, pTableMeta, CTG_GEN_STB_FLAG(isSTable))); } int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { @@ -2226,6 +2253,11 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } STableMeta *tbMeta = NULL; int32_t code = 0; @@ -2236,7 +2268,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -2307,16 +2339,21 @@ _return: } -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, false, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); @@ -2336,10 +2373,10 @@ _return: } -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2363,7 +2400,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, false, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index ebe20fbb7fb9053463b265ffd1442527e35f1e3c..f3aecadd3fef9fb3231ef720e23954058861ca67 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist); + int32_t *exist, int32_t flag); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgDbgEnableDebug(char *option); @@ -243,6 +243,8 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->suid = ctgTestSuid + 1; rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; + + rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -770,7 +772,7 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0); if (code || 0 == exist) { assert(0); } @@ -827,6 +829,7 @@ void *ctgTestSetCtableMetaThread(void *param) { #if 0 + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1289,6 +1292,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + tfree(rsp.pSchemas); while (true) { uint64_t n = 0; diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 7d9209714b51e2c26043148de01f38448e787e8b..8acb6f7e8d933153c4d35437529dfaf46acf6174 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "dataSinkMgt.h" #include "plannodes.h" diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 10d884cb3f9e1350c199832f5b8021a6b923f71e..cff7546a7eff2fc7895ac0e596ca1384e8d4ec2b 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,7 +15,7 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H -#include "common.h" +#include "tcommon.h" #include "tbuffer.h" #include "tpagedbuf.h" @@ -68,9 +68,10 @@ typedef struct SResultRow { } SResultRow; typedef struct SResultRowInfo { + SResultRow *pCurResult; // current active result row info SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set +// int16_t type:8; // data type for hash key + int32_t size; // number of result set int32_t capacity; // max capacity int32_t curPos; // current active result row index of pResult list } SResultRowInfo; @@ -95,7 +96,7 @@ struct SUdfInfo; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); size_t getResultRowSize(SArray* pExprInfo); -int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); +int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); @@ -105,7 +106,7 @@ void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 23acdf1c336adc1324113d4ffc3582f34a02cf4a..c8e1aeba0eff7cd9ff9bdf16df16fba3360160ed 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -15,12 +15,13 @@ #ifndef TDENGINE_EXECUTORIMPL_H #define TDENGINE_EXECUTORIMPL_H +#include "tsort.h" #ifdef __cplusplus extern "C" { #endif #include "os.h" -#include "common.h" +#include "tcommon.h" #include "tlosertree.h" #include "ttszip.h" #include "tvariant.h" @@ -444,16 +445,32 @@ typedef struct SOptrBasicInfo { int32_t capacity; } SOptrBasicInfo; -typedef struct SOptrBasicInfo STableIntervalOperatorInfo; - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file +typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. +} SAggSupporter; + +typedef struct STableIntervalOperatorInfo { + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SGroupResInfo groupResInfo; + SInterval interval; + STimeWindow win; + int32_t precision; + bool timeWindowInterpo; + char **pRow; + SAggSupporter aggSup; + STableQueryInfo *pCurrent; + int32_t order; +} STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SAggSupporter aggSup; STableQueryInfo *current; uint32_t groupId; SGroupResInfo groupResInfo; @@ -549,49 +566,42 @@ typedef struct SDistinctOperatorInfo { SArray* pDistinctDataInfo; } SDistinctOperatorInfo; -struct SGlobalMerger; - -typedef struct SMultiwayMergeInfo { - struct SGlobalMerger* pMerge; - SOptrBasicInfo binfo; - int32_t bufCapacity; - int64_t seed; - char** prevRow; - SArray* orderColumnList; - int32_t resultRowFactor; - - bool hasGroupColData; - char** currentGroupColData; - SArray* groupColumnList; - bool hasDataBlockForNewGroup; - SSDataBlock* pExistBlock; - - SArray* udfInfo; - bool hasPrev; - bool multiGroupResults; -} SMultiwayMergeInfo; - -typedef struct SMsortComparParam { - struct SExternalMemSource **pSources; - int32_t numOfSources; - SArray *orderInfo; // SArray - bool nullFirst; -} SMsortComparParam; +typedef struct SSortedMergeOperatorInfo { + SOptrBasicInfo binfo; + bool hasVarCol; + + SArray *orderInfo; // SArray + bool nullFirst; + int32_t numOfSources; + + SSortHandle *pSortHandle; + + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + + int32_t resultRowFactor; + bool hasGroupVal; + + SDiskbasedBuf *pTupleStore; // keep the final results + int32_t numOfResPerPage; + + char** groupVal; + SArray *groupInfo; + SAggSupporter aggSup; +} SSortedMergeOperatorInfo; typedef struct SOrderOperatorInfo { - int32_t sourceId; uint32_t sortBufSize; // max buffer size for in-memory sort SSDataBlock *pDataBlock; bool hasVarCol; // has variable length column, such as binary/varchar/nchar - int32_t numOfCompleted; - SDiskbasedBuf *pSortInternalBuf; - SMultiwayMergeTreeInfo *pMergeTree; - SArray *pSources; // SArray + SArray *orderInfo; + bool nullFirst; + SSortHandle *pSortHandle; + int32_t bufPageSize; int32_t numOfRowsInRes; - SMsortComparParam cmpParam; - + // TODO extact struct int64_t startTs; // sort start time uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. uint64_t totalSize; // total load bytes from remote @@ -608,8 +618,8 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, @@ -641,9 +651,8 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal); -SOperatorInfo* createMergeSortOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, - SOrder* pOrderVal); +SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); // SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); // SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); @@ -691,9 +700,6 @@ int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status); -bool onlyQueryTags(STaskAttr* pQueryAttr); -// void destroyUdfInfo(struct SUdfInfo* pUdfInfo); - int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen); size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows); diff --git a/include/common/tcompare.h b/source/libs/executor/inc/tlinearhash.h similarity index 50% rename from include/common/tcompare.h rename to source/libs/executor/inc/tlinearhash.h index 5dcbf37b90c03ce0a669d214544dd01da95286da..a4190582166d565d42faf062c78b87c65ccdf644 100644 --- a/include/common/tcompare.h +++ b/source/libs/executor/inc/tlinearhash.h @@ -13,28 +13,32 @@ * along with this program. If not, see . */ -#ifndef _TD_TCOMPARE_H_ -#define _TD_TCOMPARE_H_ - -#include "compare.h" -#include "ttypes.h" +#ifndef TDENGINE_TLINEARHASH_H +#define TDENGINE_TLINEARHASH_H #ifdef __cplusplus extern "C" { #endif -int32_t compareStrPatternMatch(const void* pLeft, const void* pRight); -int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight); +#include "thash.h" + +enum { + LINEAR_HASH_STATIS = 0x1, + LINEAR_HASH_DATA = 0x2, +}; + +typedef struct SLHashObj SLHashObj; -int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight); -int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight); +SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage); +void* tHashCleanup(SLHashObj* pHashObj); -__compar_fn_t getComparFunc(int32_t type, int32_t optr); -__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); -int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); +int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size); +char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen); +int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen); + +void tHashPrint(const SLHashObj* pHashObj, int32_t type); #ifdef __cplusplus } #endif - -#endif /*_TD_TCOMPARE_H_*/ +#endif // TDENGINE_TLINEARHASH_H diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h new file mode 100644 index 0000000000000000000000000000000000000000..ef7af2b4e3e08d884a5e7cc16118603746e0de02 --- /dev/null +++ b/source/libs/executor/inc/tsort.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSORT_H +#define TDENGINE_TSORT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tcommon.h" +#include "os.h" + +enum { + SORT_MULTISOURCE_MERGE = 0x1, + SORT_SINGLESOURCE_SORT = 0x2, +}; + +typedef struct SMultiMergeSource { + int32_t type; + int32_t rowIndex; + SSDataBlock *pBlock; +} SMultiMergeSource; + +typedef struct SExternalMemSource { + SMultiMergeSource src; + SArray* pageIdList; + int32_t pageIndex; +} SExternalMemSource; + +typedef struct SGenericSource { + SMultiMergeSource src; + void *param; +} SGenericSource; + +typedef struct SMsortComparParam { + void **pSources; + int32_t numOfSources; + SArray *orderInfo; // SArray + bool nullFirst; +} SMsortComparParam; + +typedef struct SSortHandle SSortHandle; +typedef struct STupleHandle STupleHandle; + +typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param); +typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param); + +/** + * + * @param type + * @return + */ +SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr); + +/** + * + * @param pSortHandle + */ +void tsortDestroySortHandle(SSortHandle* pSortHandle); + +/** + * + * @param pHandle + * @return + */ +int32_t tsortOpen(SSortHandle* pHandle); + +/** + * + * @param pHandle + * @return + */ +int32_t tsortClose(SSortHandle* pHandle); + +/** + * + * @return + */ +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp); + +/** + * + * @param pHandle + * @param fp + * @return + */ +int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); + +/** + * + * @param pHandle + * @param pSource + * @return success or failed + */ +int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource); + +/** + * + * @param pHandle + * @return + */ +STupleHandle* tsortNextTuple(SSortHandle* pHandle); + +/** + * + * @param pHandle + * @param colIndex + * @return + */ +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); + +/** + * + * @param pHandle + * @param colIndex + * @return + */ +void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSORT_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 52ab8493f1841f511a9f00b41be438899a60666f..1b901ee9f684e1894ad66128c47df6ca732126f4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -53,8 +53,7 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { return size; } -int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { - pResultRowInfo->type = type; +int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->size = 0; pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; @@ -93,7 +92,7 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRow *pWindowRes = pResultRowInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); + clearResultRow(pRuntimeEnv, pWindowRes); int32_t groupIndex = 0; int64_t uid = 0; @@ -136,7 +135,7 @@ void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { +void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { if (pResultRow == NULL) { return; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index ff2447b5957b98966124982015f2c965ffe23360..529b7f4814522598fdd9afead5e31c6e56778445 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -93,3 +93,25 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } + +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo; + + // traverse to the streamscan node to add this table id + SOperatorInfo* pInfo = pTaskInfo->pRoot; + while(pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + pInfo = pInfo->pDownstream[0]; + } + + SStreamBlockScanInfo* pScanInfo = pInfo->info; + if (isAdd) { + int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index f632a84c1f3d7c737d60fdaa4149eaf9ea896a26..1642120d150148da8b5ac5455accebd934810ddd 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -15,7 +15,7 @@ #include #include "dataSinkMgt.h" -#include "exception.h" +#include "texception.h" #include "os.h" #include "tarray.h" #include "tcache.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6d983700e893039cba447b0c00ff569087c5ff69..8cae5f8768269bed59f7b6253a8c10df5f56032b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,13 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "exception.h" + #include "os.h" + +#include "tep.h" +#include "tsort.h" +#include "texception.h" #include "parser.h" #include "tglobal.h" #include "tmsg.h" -#include "tq.h" #include "ttime.h" #include "executorimpl.h" @@ -140,19 +142,19 @@ static int32_t getExprFunctionId(SExprInfo *pExprInfo) { return 0; } -static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { - tw->skey += pQueryAttr->interval.sliding * factor; - tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; +static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = tw->skey + pInterval->interval - 1; return; } - int64_t key = tw->skey, interval = pQueryAttr->interval.interval; + int64_t key = tw->skey, interval = pInterval->interval; //convert key to second - key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - if (pQueryAttr->interval.intervalUnit == 'y') { + if (pInterval->intervalUnit == 'y') { interval *= 12; } @@ -163,12 +165,12 @@ static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); tw->ekey -= 1; } @@ -178,7 +180,7 @@ static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult int32_t numOfCols, int32_t* rowCellInfoOffset); void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx); +static bool functionNeedToExecute(SqlFunctionCtx *pCtx); static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); @@ -221,7 +223,8 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); + static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); @@ -380,18 +383,23 @@ static bool hasNull(SColumn* pColumn, SColumnDataAgg *pStatis) { } static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { + int64_t newCapacity = 0; + // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; } - int64_t newCapacity = 0; if (pResultRowInfo->capacity > 10000) { newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); } else { newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); } + if (newCapacity == pResultRowInfo->capacity) { + newCapacity += 4; + } + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -518,12 +526,12 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR } static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, - bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggOperatorInfo* pAggInfo) { + bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { bool existed = false; - SET_RES_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tableGroupId); + SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId); SResultRow **p1 = - (SResultRow **)taosHashGet(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRow **)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); // in case of repeat scan/reverse scan, no new time window added. if (isIntervalQuery) { @@ -539,8 +547,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int existed = (pResultRowInfo->pResult[0] == (*p1)); pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; existed = true; @@ -561,16 +569,16 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int SResultRow *pResult = NULL; if (p1 == NULL) { - pResult = getNewResultRow(pAggInfo->pool); + pResult = getNewResultRow(pSup->pool); int32_t ret = initResultRow(pResult); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // add a new result set for a new group - taosHashPut(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pAggInfo->pResultRowArrayList, &cell); + taosArrayPush(pSup->pResultRowArrayList, &cell); } else { pResult = *p1; } @@ -579,8 +587,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; int64_t index = pResultRowInfo->curPos; - SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); } // too many time window in query @@ -591,19 +599,19 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int return pResultRowInfo->pResult[pResultRowInfo->curPos]; } -static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, precision, ts, ts, ekey, w); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); + getAlignQueryTimeWindow(pInterval, precision, ts, ekey, ts, w); int64_t key = w->skey; while(key < ts) { // moving towards end - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); } else { - key += pQueryAttr->interval.sliding; + key += pInterval->sliding; } if (key >= ts) { @@ -616,39 +624,39 @@ static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWind } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); + getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; + w.ekey = w.skey + pInterval->interval - 1; } } else { w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; } if (w.skey > ts || w.ekey < ts) { - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { int64_t st = w.skey; if (st > ts) { - st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; } - int64_t et = st + pQueryAttr->interval.interval - 1; + int64_t et = st + pInterval->interval - 1; if (et < ts) { - st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; } w.skey = st; - w.ekey = w.skey + pQueryAttr->interval.interval - 1; + w.ekey = w.skey + pInterval->interval - 1; } } @@ -656,10 +664,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t * query border check, skey should not be bounded by the query time range, since the value skey will * be used as the time window index value. So we only change ekey of time window accordingly. */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - +// ASSERT(win->skey <= win->ekey); // todo no need this return w; } @@ -668,7 +673,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, i STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); +// getInitialStartTimeWindow(pQueryAttr, ts, &w); if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; @@ -703,7 +708,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes SIDList list = getDataBufPagesIdList(pResultBuf, tid); if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, tid, &pageId); + pData = getNewBufPage(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); pData = getBufPage(pResultBuf, getPageId(pi)); @@ -712,7 +717,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes if (pData->num + size > getBufPageSize(pResultBuf)) { // release current page first, and prepare the next one releaseBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); + pData = getNewBufPage(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer } @@ -770,6 +775,37 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowI return TSDB_CODE_SUCCESS; } +static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); + +static int32_t setResultOutputBufByKey_rv(SResultRowInfo *pResultRowInfo, int64_t id, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset, SDiskbasedBuf *pBuf, SAggSupporter *pAggSup, SExecTaskInfo* pTaskInfo) { + assert(win->skey <= win->ekey); + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pResultRowInfo, id, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, + pTaskInfo, true, pAggSup); + + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // not assign result buffer yet, add new result buffer + if (pResultRow->pageId == -1) { // todo intermediate result size + int32_t ret = addNewWindowResultBuf(pResultRow, pBuf, (int32_t) tableGroupId, 0); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + + return TSDB_CODE_SUCCESS; +} + + static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { @@ -871,48 +907,44 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, STaskAttr* pQueryAttr, TSKEY lastKey) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, bool ascQuery, bool interp) { + if ((lastKey > pWin->ekey && ascQuery) || (lastKey < pWin->ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); pResultRowInfo->curPos = pResultRowInfo->size - 1; } else { int32_t step = ascQuery ? 1 : -1; - doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, interp); } } -static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { +static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo* item = pRuntimeEnv->current; int32_t num = -1; - int32_t order = pQueryAttr->order.order; int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (order == TSDB_ORDER_ASC) { if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key + if (item != NULL) { item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; } } else { num = pDataBlockInfo->rows - startPos; - if (updateLastKey) { + if (item != NULL) { item->lastKey = pDataBlockInfo->window.ekey + step; } } } else { // desc if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key + if (item != NULL) { item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; } } else { num = startPos + 1; - if (updateLastKey) { + if (item != NULL) { item->lastKey = pDataBlockInfo->window.skey + step; } } @@ -922,22 +954,18 @@ static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlock return num; } -static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasAggregates = pCtx[0].isAggSet; - +static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].size = forwardStep; pCtx[k].startTs = pWin->skey; // keep it temporarialy - char* start = pCtx[k].pInput; + int32_t startOffset = pCtx[k].startRow; + bool hasAgg = pCtx[k].isAggSet; - int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); - if (pCtx[k].pInput != NULL) { - pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; - } + int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); + pCtx[k].startRow = pos; if (tsCol != NULL) { pCtx[k].ptsList = &tsCol[pos]; @@ -949,88 +977,80 @@ static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, pCtx[k].isAggSet = false; } - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + if (functionNeedToExecute(&pCtx[k])) { pCtx[k].fpSet->addInput(&pCtx[k]); } // restore it - pCtx[k].isAggSet = hasAggregates; - pCtx[k].pInput = start; + pCtx[k].isAggSet = hasAgg; + pCtx[k].startRow = startOffset; } } -static int32_t getNextQualifiedWindow(STaskAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - getNextTimeWindow(pQueryAttr, pNext); +static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { + int32_t order = pInfo->order; + bool ascQuery = (order == TSDB_ORDER_ASC); + + int32_t precision = pInfo->precision; + getNextTimeWindow(pInterval, precision, order, pNext); // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || - (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || + (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { return -1; } - TSKEY startKey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - startKey = pNext->skey; - if (startKey < pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } else { - startKey = pNext->ekey; - if (startKey > pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } - + TSKEY startKey = ascQuery? pNext->skey:pNext->ekey; int32_t startPos = 0; // tumbling time window query, a special case of sliding time window query - if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + if (pInterval->sliding == pInterval->interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); startPos = prevPosition + factor; } else { - if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (startKey <= pDataBlockInfo->window.skey && ascQuery) { startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { + } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { startPos = pDataBlockInfo->rows - 1; } else { - startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); + startPos = binarySearchForKey((char *)primaryKeys, pDataBlockInfo->rows, startKey, order); } } /* interp query with fill should not skip time window */ - if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - return startPos; - } +// if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { +// return startPos; +// } /* * This time window does not cover any data, try next time window, * this case may happen when the time window is too small */ if (primaryKeys == NULL) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (ascQuery) { assert(pDataBlockInfo->window.skey <= pNext->ekey); } else { assert(pDataBlockInfo->window.ekey >= pNext->skey); } } else { - if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { + if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; + pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1)/pInterval->sliding) * pInterval->sliding; + pNext->skey = pNext->ekey - pInterval->interval + 1; } - } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { + } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; + pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->ekey = pNext->skey + pInterval->interval - 1; } } } @@ -1067,23 +1087,19 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static void saveDataBlockLastRow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, - int32_t rowIndex) { +static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { if (pDataBlock == NULL) { return; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { + for (int32_t k = 0; k < numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } -static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { TSKEY ts = TSKEY_INITIAL_VAL; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); if (tsCols == NULL) { ts = ascQuery? win->skey : win->ekey; } else { @@ -1151,7 +1167,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pCtx[i].columnIndex); // in case of the block distribution query, the inputBytes is not a constant value. - pCtx[i].pInput = p->pData; + pCtx[i].pInput = p; assert(p->info.colId == pCol->info.colId); if (pCtx[i].functionId < 0) { @@ -1163,14 +1179,14 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, // uint32_t status = aAggs[pCtx[i].functionId].status; // if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); +// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); // In case of the top/bottom query again the nest query result, which has no timestamp column // don't set the ptsList attribute. - if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - } else { - pCtx[i].ptsList = NULL; - } +// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { +// pCtx[i].ptsList = (int64_t*) tsInfo->pData; +// } else { +// pCtx[i].ptsList = NULL; +// } // } // } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { // SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; @@ -1189,7 +1205,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - if (functionNeedToExecute(NULL, &pCtx[k])) { + if (functionNeedToExecute(&pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct pCtx[k].fpSet->addInput(&pCtx[k]); } @@ -1348,20 +1364,19 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun } static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - if (!pQueryAttr->timeWindowInterpo) { + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, int32_t order, bool timeWindowInterpo) { + if (!timeWindowInterpo) { return; } assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); if (pBlock->pDataBlock == NULL){ // tscError("pBlock->pDataBlock == NULL"); return; } + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); TSKEY *tsCols = (TSKEY *)(pColInfo->pData); @@ -1374,38 +1389,37 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); } // point interpolation does not require the end key time window interpolation. - if (pQueryAttr->pointInterpQuery) { - return; - } +// if (pointInterpQuery) { +// return; +// } // interpolation query does not generate the time window end interpolation done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); if (!done) { int32_t endRowIndex = startPos + (forwardStep - 1) * step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; + TSKEY endKey = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); } } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfOutput; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + int32_t step = 1; + bool ascQuery = true; int32_t prevIndex = pResultRowInfo->curPos; @@ -1418,26 +1432,26 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->precision, &pInfo->win); + bool masterScan = true; SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); + int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } int32_t forwardStep = 0; - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + TSKEY ekey = win.ekey; forwardStep = - getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // prev time window not interpolation yet. int32_t curIndex = pResultRowInfo->curPos; - if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { + if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. SResultRow* pRes = getResultRow(pResultRowInfo, j); if (pRes->closed) { @@ -1446,64 +1460,64 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } STimeWindow w = pRes->win; - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + doApplyFunctions(pInfo->binfo.pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } // restore current time window - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); + ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); + doApplyFunctions(pInfo->binfo.pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); if (startPos < 0) { break; } // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - ekey = reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + ekey = nextWin.ekey;//reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } - if (pQueryAttr->timeWindowInterpo) { + if (pInfo->timeWindowInterpo) { int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); } - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +// updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); } @@ -1526,7 +1540,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); @@ -1539,25 +1553,25 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe while (1) { // null data, failed to allocate more memory buffer ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); +// forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); +// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); preWin = win; int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); +// startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); if (startPos < 0) { if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1565,8 +1579,8 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe startPos = pSDataBlock->info.rows - 1; // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); +// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); } break; @@ -1576,10 +1590,10 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pQueryAttr->timeWindowInterpo) { int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); +// saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); } - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +// updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } @@ -1641,7 +1655,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); num = 1; memcpy(pInfo->prevData, val, bytes); @@ -1660,7 +1674,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); } tfree(pInfo->prevData); @@ -1710,8 +1724,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); pInfo->curWindow.skey = tsList[j]; pInfo->curWindow.ekey = tsList[j]; @@ -1731,8 +1745,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { @@ -1806,7 +1820,7 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx) { +static bool functionNeedToExecute(SqlFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // in case of timestamp column, always generated results. @@ -2046,6 +2060,10 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC pCtx->resDataInfo.type = pSqlExpr->resSchema.type; pCtx->order = TSDB_ORDER_ASC; + if (i == 0) { + pCtx->functionId = FUNCTION_TS; + } + // pCtx->functionId = pSqlExpr->functionId; // pCtx->stableQuery = pQueryAttr->stableQuery; pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; @@ -2214,7 +2232,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); // destroyUdfInfo(pRuntimeEnv->pUdfInfo); - destroyResultBuf(pRuntimeEnv->pResultBuf); + destroyDiskbasedBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); destroyTsComp(pRuntimeEnv, pQueryAttr); @@ -2340,46 +2358,23 @@ static bool isCachedLastQuery(STaskAttr *pQueryAttr) { return true; } - - -/** - * The following 4 kinds of query are treated as the tags query - * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query - */ -bool onlyQueryTags(STaskAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; - - int32_t functionId = getExprFunctionId(pExprInfo); - - if (functionId != FUNCTION_TAGPRJ && - functionId != FUNCTION_TID_TAG && - (!(functionId == FUNCTION_COUNT && pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.pColumns->flag)))) { - return false; - } - } - - return true; -} - ///////////////////////////////////////////////////////////////////////////////////////////// - -void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); - win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); +//todo refactor : return window +void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { + assert(key >= keyFirst && key <= keyLast && pInterval->sliding <= pInterval->interval); + win->skey = taosTimeTruncate(key, pInterval, precision); /* - * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between + * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { - assert(keyLast - keyFirst < pQueryAttr->interval.interval); + if (keyFirst > (INT64_MAX - pInterval->interval)) { + assert(keyLast - keyFirst < pInterval->interval); win->ekey = INT64_MAX; - } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - win->ekey = win->skey + pQueryAttr->interval.interval - 1; + win->ekey = win->skey + pInterval->interval - 1; } } @@ -2605,7 +2600,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); assert(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -2613,7 +2608,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } while(1) { - getNextTimeWindow(pQueryAttr, &w); +// getNextTimeWindow(pQueryAttr, &w); if (w.skey > pBlockInfo->window.ekey) { break; } @@ -2624,7 +2619,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } } } else { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); assert(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { @@ -2632,7 +2627,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } while(1) { - getNextTimeWindow(pQueryAttr, &w); +// getNextTimeWindow(pQueryAttr, &w); if (w.ekey < pBlockInfo->window.skey) { break; } @@ -2863,8 +2858,6 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, tfree(p); } - - static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); @@ -3379,10 +3372,8 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } - -void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int32_t stage, SExecTaskInfo* pTaskInfo) { - SOptrBasicInfo *pInfo = &pAggInfo->binfo; - +// TODO refactor: some function move away +void setDefaultOutputBuf_rv(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, SExecTaskInfo* pTaskInfo) { SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; @@ -3390,9 +3381,7 @@ void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int32_t stage, SExecTask int64_t tid = 0; int64_t groupId = 0; - - pAggInfo->keyBuf = realloc(pAggInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pAggInfo); + SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3540,9 +3529,6 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCt } void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperator->numOfOutput; // if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { // // for each group result, call the finalize function for each column @@ -3605,7 +3591,7 @@ STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow // set more initial size of interval/groupby query // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { int32_t initialSize = 128; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -3623,7 +3609,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { // set more initial size of interval/groupby query int32_t initialSize = 16; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { tfree(pTableQueryInfo); return NULL; @@ -3716,7 +3702,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; SResultRow* pResultRow = - doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, pAggInfo); + doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, &pAggInfo->aggSup); assert (pResultRow != NULL); /* @@ -3935,7 +3921,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { TSKEY sk = TMIN(win.skey, win.ekey); TSKEY ek = TMAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { @@ -4520,18 +4506,23 @@ void queryCostStatis(SExecTaskInfo *pTaskInfo) { // return true; //} -void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { +int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { if (p->pDownstream == NULL) { assert(p->numOfDownstream == 0); } - p->pDownstream = realloc(p->pDownstream, POINTER_BYTES * (p->numOfDownstream + 1)); - p->pDownstream[p->numOfDownstream++] = pUpstream; + p->pDownstream = calloc(1, num * POINTER_BYTES); + if (p->pDownstream == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES); + p->numOfDownstream = num; + return TSDB_CODE_SUCCESS; } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); -void createResultBlock(const SArray* pExprInfo, SExchangeInfo* pInfo, const SOperatorInfo* pOperator, size_t size); static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 @@ -4624,11 +4615,11 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); } - int32_t ps = DEFAULT_PAGE_SIZE; + int32_t ps = 4096; getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp"); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5441,13 +5432,17 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); for(int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId); } // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); + int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); + if (code != 0) { + tfree(pInfo); + tfree(pOperator); + return NULL; + } pInfo->readerHandle = streamReadHandle; @@ -5462,7 +5457,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp return pOperator; } - void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { assert(pTableScanInfo != NULL && pDownstream != NULL); @@ -5595,14 +5589,20 @@ SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { return pOrderColumns; } -static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; - destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); +static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo); +static void clearupAggSup(SAggSupporter* pAggSup); - taosArrayDestroy(pInfo->orderColumnList); - taosArrayDestroy(pInfo->groupColumnList); - tfree(pInfo->prevRow); - tfree(pInfo->currentGroupColData); +static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { + SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*) param; + taosArrayDestroy(pInfo->orderInfo); + taosArrayDestroy(pInfo->groupInfo); + + if (pInfo->pSortHandle != NULL) { + tsortDestroySortHandle(pInfo->pSortHandle); + } + blockDataDestroy(pInfo->binfo.pRes); + + clearupAggSup(&pInfo->aggSup); } static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { @@ -5612,487 +5612,411 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevRow); } -SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, - SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->resultRowFactor = - (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); +static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { + assert(dst != NULL && src != NULL); - pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx + *dst = *src; - pInfo->multiGroupResults = groupResultMixedUp; - pInfo->pMerge = param; - pInfo->bufCapacity = 4096; - pInfo->udfInfo = pUdfInfo; - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); - pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); + dst->pExpr = exprdup(src->pExpr); + dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); + memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { -// len += pExpr[i].base.; + memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); + for (int32_t j = 0; j < src->base.numOfParams; ++j) { + taosVariantAssign(&dst->base.param[j], &src->base.param[j]); } +} - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - int32_t offset = POINTER_BYTES * numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; +static SExprInfo* exprArrayDup(SArray* pExprList) { + size_t numOfOutput = taosArrayGetSize(pExprList); - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resSchema.bytes; + SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprList, i); + assignExprInfo(&p[i], pExpr); } - numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; - pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); - offset = POINTER_BYTES * numOfCols; + return p; +} - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; +// TODO merge aggregate super table +static void appendOneRowToDataBlock(SSDataBlock *pBlock, STupleHandle* pTupleHandle) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); - offset += pExpr[index->colIndex].base.resSchema.bytes; + bool isNull = tsortIsNullVal(pTupleHandle, i); + if (isNull) { + colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + } else { + char* pData = tsortGetValue(pTupleHandle, i); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } } - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); - - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); - - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GlobalAggregate"; -// pOperator->operatorType = OP_GlobalAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; -// pOperator->exec = doGlobalAggregate; - pOperator->cleanupFn = destroyGlobalAggOperatorInfo; - appendDownstream(pOperator, downstream); - - return pOperator; + pBlock->info.rows += 1; } -SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, - int32_t numOfRows, void *merger) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); +static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, int32_t capacity) { + blockDataClearup(pDataBlock, hasVarCol); - pInfo->pMerge = merger; - pInfo->bufCapacity = numOfRows; - pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); - - { // todo extract method to create prev compare buffer - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { -// len += pExpr[i].base.colBytes; + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; } - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; - - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); -// offset += pExpr[index->colIndex].base.colBytes; + appendOneRowToDataBlock(pDataBlock, pTupleHandle); + if (pDataBlock->info.rows >= capacity) { + return pDataBlock; } } - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiwaySortOperator"; -// pOperator->operatorType = OP_MultiwayMergeSort; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; -// pOperator->exec = doMultiwayMergeSort; - pOperator->cleanupFn = destroyGlobalAggOperatorInfo; - return pOperator; + return (pDataBlock->info.rows > 0)? pDataBlock:NULL; +} + +SSDataBlock* loadNextDataBlock(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + bool newgroup = false; + + return pOperator->exec(pOperator, &newgroup); } -typedef struct SExternalMemSource { - SArray* pageIdList; - int32_t pageIndex; - int32_t sourceId; - int32_t rowIndex; - SSDataBlock *pBlock; -} SExternalMemSource; +static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) { + size_t size = taosArrayGetSize(groupInfo); + if (size == 0) { + return true; + } + + for (int32_t i = 0; i < size; ++i) { + int32_t* index = taosArrayGet(groupInfo, i); -int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t pLeftIdx = *(int32_t *)pLeft; - int32_t pRightIdx = *(int32_t *)pRight; + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); + bool isNull = colDataIsNull(pColInfo, rowIndex, pBlock->info.rows, NULL); - SMsortComparParam *pParam = (SMsortComparParam *)param; + if ((isNull && buf[i] != NULL) || (!isNull && buf[i] == NULL)) { + return false; + } - SArray *pInfo = pParam->orderInfo; + char* pCell = colDataGetData(pColInfo, rowIndex); + if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + if (varDataLen(pCell) != varDataLen(buf[i])) { + return false; + } else { + if (memcmp(varDataVal(pCell), varDataVal(buf[i]), varDataLen(pCell)) != 0) { + return false; + } + } + } else { + if (memcmp(pCell, buf[i], pColInfo->info.bytes) != 0) { + return false; + } + } + } - SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; - SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; + return 0; +} - // this input is exhausted, set the special value to denote this - if (pLeftSource->rowIndex == -1) { - return 1; +static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx *pCtx, int32_t numOfExpr, int32_t rowIndex) { + for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index + pCtx[j].startRow = rowIndex; } - if (pRightSource->rowIndex == -1) { - return -1; + for (int32_t j = 0; j < numOfExpr; ++j) { + int32_t functionId = pCtx[j].functionId; + pCtx[j].fpSet->addInput(&pCtx[j]); + +// if (functionId < 0) { +// SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); +// doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); +// } else { +// assert(!TSDB_FUNC_IS_SCALAR(functionId)); +// aAggs[functionId].mergeFunc(&pCtx[j]); +// } } +} - SSDataBlock* pLeftBlock = pLeftSource->pBlock; - SSDataBlock* pRightBlock = pRightSource->pBlock; +static void doFinalizeResultImpl(SqlFunctionCtx *pCtx, int32_t numOfExpr) { + for(int32_t j = 0; j < numOfExpr; ++j) { + int32_t functionId = pCtx[j].functionId; + // if (functionId == FUNC_TAG_DUMMY || functionId == FUNC_TS_DUMMY) { + // continue; + // } - for(int32_t i = 0; i < pInfo->size; ++i) { - SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + // if (functionId < 0) { + // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); + // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); + // } else { + pCtx[j].fpSet->addInput(&pCtx[j]); + } +} - SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); +static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) { + int32_t size = (int32_t) taosArrayGetSize(pColumnList); - bool leftNull = false; - if (pLeftColInfoData->hasNull) { - leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->rowIndex, pLeftBlock->pBlockAgg); - } + for(int32_t i = 0; i < size; ++i) { + int32_t* index = taosArrayGet(pColumnList, i); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); - SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); - bool rightNull = false; - if (pRightColInfoData->hasNull) { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->rowIndex, pRightBlock->pBlockAgg); - } + char* data = colDataGetData(pColInfo, rowIndex); + memcpy(rowColData[i], data, colDataGetLength(pColInfo, rowIndex)); + } - if (leftNull && rightNull) { - continue; // continue to next slot - } + return true; +} - if (rightNull) { - return pParam->nullFirst? 1:-1; - } +static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) { + SSortedMergeOperatorInfo* pInfo = pOperator->info; - if (leftNull) { - return pParam->nullFirst? -1:1; - } + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + pCtx[i].size = 1; + } - void* left1 = colDataGet(pLeftColInfoData, pLeftSource->rowIndex); - void* right1 = colDataGet(pRightColInfoData, pRightSource->rowIndex); + for(int32_t i = 0; i < pBlock->info.rows; ++i) { + if (!pInfo->hasGroupVal) { + ASSERT(i == 0); + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); + } else { + if (needToMerge(pBlock, pInfo->groupInfo, pInfo->groupVal, i)) { + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + } else { + doFinalizeResultImpl(pCtx, numOfExpr); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); - switch(pLeftColInfoData->info.type) { - case TSDB_DATA_TYPE_INT: { - int32_t leftv = *(int32_t*)left1; - int32_t rightv = *(int32_t*)right1; + // TODO check for available buffer; - if (leftv == rightv) { - break; - } else { - if (pOrder->order == TSDB_ORDER_ASC) { - return leftv < rightv? -1 : 1; - } else { - return leftv < rightv? 1 : -1; + // next group info data + pInfo->binfo.pRes->info.rows += numOfRows; + for (int32_t j = 0; j < numOfExpr; ++j) { + if (pCtx[j].functionId < 0) { + continue; } + + pCtx[j].fpSet->addInput(&pCtx[j]); } + + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); } - default: - assert(0); } } } -static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SOrderOperatorInfo* pInfo) { - /* - * load a new SDataBlock into memory of a given intermediate data-set source, - * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree - */ - if (pSource->rowIndex >= pSource->pBlock->info.rows) { - pSource->rowIndex = 0; - pSource->pageIndex += 1; - - if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { - pInfo->numOfCompleted += 1; - pSource->rowIndex = -1; - pSource->pageIndex = -1; - pSource->pBlock = blockDataDestroy(pSource->pBlock); - } else { - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); +static SSDataBlock* doMerge(SOperatorInfo* pOperator) { + SSortedMergeOperatorInfo* pInfo = pOperator->info; + SSortHandle* pHandle = pInfo->pSortHandle; - SFilePage* pPage = getBufPage(pInfo->pSortInternalBuf, getPageId(pPgInfo)); - int32_t code = blockDataFromBuf(pSource->pBlock, pPage->data); - if (code != TSDB_CODE_SUCCESS) { - return code; + SSDataBlock* pDataBlock = createOneDataBlock(pInfo->binfo.pRes); + blockDataEnsureCapacity(pDataBlock, pInfo->binfo.capacity); + + while(1) { + + blockDataClearup(pDataBlock, pInfo->hasVarCol); + while (1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; } - releaseBufPage(pInfo->pSortInternalBuf, pPage); + // build datablock for merge for one group + appendOneRowToDataBlock(pDataBlock, pTupleHandle); + if (pDataBlock->info.rows >= pInfo->binfo.capacity) { + break; + } } - } - /* - * Adjust loser tree otherwise, according to new candidate data - * if the loser tree is rebuild completed, we do not need to adjust - */ - int32_t leafNodeIndex = tMergeTreeGetAdjustIndex(pTree); + if (pDataBlock->info.rows == 0) { + break; + } -#ifdef _DEBUG_VIEW - printf("before adjust:\t"); - tMergeTreePrint(pTree); -#endif + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC); + // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, pOperator->pRuntimeEnv, true); + doMergeImpl(pOperator, pOperator->numOfOutput, pDataBlock); + // flush to tuple store, and after all data have been handled, return to upstream node or sink node + } - tMergeTreeAdjust(pTree, leafNodeIndex); + doFinalizeResultImpl(pInfo->binfo.pCtx, pOperator->numOfOutput); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); -#ifdef _DEBUG_VIEW - printf("\nafter adjust:\t"); - tMergeTreePrint(pTree); -#endif -} + // TODO check for available buffer; -static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + // next group info data + pInfo->binfo.pRes->info.rows += numOfRows; + return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL; +} - SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); - bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); - - if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); - } else { - char* pData = colDataGet(pSrcColInfo, *rowIndex); - colDataAppend(pColInfo, pBlock->info.rows, pData, false); - } - } - - pBlock->info.rows += 1; - *rowIndex += 1; -} - -static int32_t doAddNewSource(SOrderOperatorInfo* pInfo, SArray* pAllSources, int32_t numOfCols) { - SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); - if (pSource == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; +static SSDataBlock* doSortedMerge(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - pSource->pageIdList = getDataBufPagesIdList(pInfo->pSortInternalBuf, pInfo->sourceId); - pSource->sourceId = pInfo->sourceId; - - pSource->pBlock = calloc(1, sizeof(SSDataBlock)); - pSource->pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pSource->pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - SColumnInfoData* p = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); - colInfo.info = p->info; - taosArrayPush(pSource->pBlock->pDataBlock, &colInfo); + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSortedMergeOperatorInfo* pInfo = pOperator->info; + if (pOperator->status == OP_RES_TO_RETURN) { + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->hasVarCol, pInfo->binfo.capacity); } - taosArrayPush(pAllSources, &pSource); - - pInfo->sourceId += 1; - - int32_t rowSize = blockDataGetSerialRowSize(pSource->pBlock); - int32_t numOfRows = (getBufPageSize(pInfo->pSortInternalBuf) - blockDataGetSerialMetaSize(pInfo->pDataBlock))/rowSize; - - return blockDataEnsureCapacity(pSource->pBlock, numOfRows); -} - -void addToDiskbasedBuf(SOrderOperatorInfo* pInfo, SArray* pSources, jmp_buf env) { - int32_t start = 0; + SSchema* p = blockDataExtractSchema(pInfo->binfo.pRes, NULL); + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, + numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)"); - while(start < pInfo->pDataBlock->info.rows) { - int32_t stop = 0; - blockDataSplitRows(pInfo->pDataBlock, pInfo->hasVarCol, start, &stop, getBufPageSize(pInfo->pSortInternalBuf)); - SSDataBlock* p = blockDataExtractBlock(pInfo->pDataBlock, start, stop - start + 1); - if (p == NULL) { - longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - int32_t pageId = -1; - SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); - if (pPage == NULL) { - assert(0); - longjmp(env, terrno); - } - - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); - assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); - - blockDataToBuf(pPage->data, p); - - setBufPageDirty(pPage, true); - releaseBufPage(pInfo->pSortInternalBuf, pPage); + tfree(p); + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - blockDataDestroy(p); - start = stop + 1; + for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + ps->param = pOperator->pDownstream[i]; + tsortAddSource(pInfo->pSortHandle, ps); } - int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; - blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); - - int32_t code = doAddNewSource(pInfo, pSources, numOfCols); + int32_t code = tsortOpen(pInfo->pSortHandle); if (code != TSDB_CODE_SUCCESS) { - longjmp(env, code); + longjmp(pTaskInfo->env, terrno); } + + pOperator->status = OP_RES_TO_RETURN; + return doMerge(pOperator); } -static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SDiskbasedBuf* pBuf) { - cmpParam->pSources = taosArrayGet(pSources, startIndex); - cmpParam->numOfSources = (endIndex - startIndex + 1); +static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { + SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; - SPageInfo* pPgInfo = *(SPageInfo**) taosArrayGet(pSource->pageIdList, pSource->pageIndex); + size_t numOfOrder = taosArrayGetSize(pOrderVal); + for (int32_t j = 0; j < numOfOrder; ++j) { + SBlockOrderInfo orderInfo = {0}; + SOrder* pOrder = taosArrayGet(pOrderVal, j); + orderInfo.order = pOrder->order; - SFilePage* pPage = getBufPage(pBuf, getPageId(pPgInfo)); - int32_t code = blockDataFromBuf(cmpParam->pSources[i]->pBlock, pPage->data); - if (code != TSDB_CODE_SUCCESS) { - return code; + for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + if (pExpr->base.resSchema.colId == pOrder->col.info.colId) { + orderInfo.colIndex = i; + break; + } } - releaseBufPage(pBuf, pPage); + taosArrayPush(pOrderInfo, &orderInfo); } - return TSDB_CODE_SUCCESS; + return pOrderInfo; } -static int32_t sortComparClearup(SMsortComparParam* cmpParam) { - for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; - blockDataDestroy(pSource->pBlock); - tfree(pSource); +static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeOperatorInfo* pInfo) { + if (pGroupInfo == NULL || taosArrayGetSize(pGroupInfo) == 0) { + return 0; } - cmpParam->numOfSources = 0; -} - -static SSDataBlock* getSortedBlockData(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo, SMsortComparParam* cmpParam, int32_t capacity) { - blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); - - while(1) { - if (cmpParam->numOfSources == pInfo->numOfCompleted) { - break; - } - - int32_t index = tMergeTreeGetChosenIndex(pInfo->pMergeTree); - - SExternalMemSource *pSource = (*cmpParam).pSources[index]; - appendOneRowToDataBlock(pInfo->pDataBlock, pSource->pBlock, &pSource->rowIndex); - - int32_t code = adjustMergeTreeForNextTuple(pSource, pInfo->pMergeTree, pInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - if (pInfo->pDataBlock->info.rows >= capacity) { - return pInfo->pDataBlock; + int32_t len = 0; + SArray* plist = taosArrayInit(3, sizeof(SColumn)); + pInfo->groupInfo = taosArrayInit(3, sizeof(int32_t)); + + if (plist == NULL || pInfo->groupInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + size_t numOfGroupCol = taosArrayGetSize(pInfo->groupInfo); + for(int32_t i = 0; i < numOfGroupCol; ++i) { + SColumn* pCol = taosArrayGet(pGroupInfo, i); + for(int32_t j = 0; j < taosArrayGetSize(pExprInfo); ++j) { + SExprInfo* pe = taosArrayGet(pExprInfo, j); + if (pe->base.resSchema.colId == pCol->info.colId) { + taosArrayPush(plist, pCol); + taosArrayPush(pInfo->groupInfo, &j); + len += pCol->info.bytes; + break; + } } } - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; -} - -static int32_t doInternalSort(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo) { - size_t numOfSources = taosArrayGetSize(pInfo->pSources); - - // Calculate the I/O counts to complete the data sort. - double sortCount = floorl(log2(numOfSources) / log2(getNumOfInMemBufPages(pInfo->pSortInternalBuf))); - - pInfo->totalElapsed = taosGetTimestampUs() - pInfo->startTs; - qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, - GET_TASKID(pTaskInfo), (int32_t) (sortCount + 1), getTotalBufSize(pInfo->pSortInternalBuf), pInfo->sortElapsed, - pInfo->totalElapsed); - - size_t pgSize = getBufPageSize(pInfo->pSortInternalBuf); - int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pInfo->pDataBlock))/ blockDataGetSerialRowSize(pInfo->pDataBlock); - - blockDataEnsureCapacity(pInfo->pDataBlock, numOfRows); + ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - size_t numOfSorted = taosArrayGetSize(pInfo->pSources); - for(int32_t t = 0; t < sortCount; ++t) { - int64_t st = taosGetTimestampUs(); - - SArray* pResList = taosArrayInit(4, POINTER_BYTES); - SMsortComparParam resultParam = {.orderInfo = pInfo->cmpParam.orderInfo}; - - int32_t numOfInputSources = getNumOfInMemBufPages(pInfo->pSortInternalBuf); - int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources; - - // Only *numOfInputSources* can be loaded into buffer to perform the external sort. - for(int32_t i = 0; i < sortGroup; ++i) { - pInfo->sourceId += 1; - - int32_t end = (i + 1) * numOfInputSources - 1; - if (end > numOfSorted - 1) { - end = numOfSorted - 1; - } - - pInfo->cmpParam.numOfSources = end - i * numOfInputSources + 1; - - int32_t code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, i * numOfInputSources, end, pInfo->pSortInternalBuf); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len)); + if (pInfo->groupVal == NULL) { + taosArrayDestroy(plist); + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + int32_t offset = 0; + char *start = (char*)(pInfo->groupVal + (POINTER_BYTES * numOfGroupCol)); + for(int32_t i = 0; i < numOfGroupCol; ++i) { + pInfo->groupVal[i] = start + offset; + SColumn* pCol = taosArrayGet(plist, i); + offset += pCol->info.bytes; + } - while (1) { - SSDataBlock* pDataBlock = getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, numOfRows); - if (pDataBlock == NULL) { - break; - } + taosArrayDestroy(plist); - int32_t pageId = -1; - SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); - if (pPage == NULL) { - assert(0); - longjmp(pTaskInfo->env, terrno); - } + return TSDB_CODE_SUCCESS; +} - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); - assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { + SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } - blockDataToBuf(pPage->data, pDataBlock); + int32_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - setBufPageDirty(pPage, true); - releaseBufPage(pInfo->pSortInternalBuf, pPage); + if (pInfo->binfo.pCtx == NULL || pInfo->binfo.pRes == NULL) { + goto _error; + } - blockDataClearup(pDataBlock, pInfo->hasVarCol); - } + int32_t code = initAggSup(&pInfo->aggSup, pExprInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - tMergeTreeDestroy(pInfo->pMergeTree); - pInfo->numOfCompleted = 0; + setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); + code = initGroupCol(pExprInfo, pGroupInfo, pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - code = doAddNewSource(pInfo, pResList, pInfo->pDataBlock->info.numOfCols); - if (code != 0) { - longjmp(pTaskInfo->env, code); - } - } +// pInfo->resultRowFactor = (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, +// pRuntimeEnv->pQueryAttr->topBotQuery, false)); + pInfo->sortBufSize = 1024 * 16; // 1MB + pInfo->bufPageSize = 1024; + pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); - sortComparClearup(&pInfo->cmpParam); + pInfo->binfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); - taosArrayClear(pInfo->pSources); - taosArrayAddAll(pInfo->pSources, pResList); - taosArrayDestroy(pResList); + pOperator->name = "SortedMerge"; + // pOperator->operatorType = OP_SortedMerge; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = exprArrayDup(pExprInfo); - pInfo->cmpParam = resultParam; - numOfSorted = taosArrayGetSize(pInfo->pSources); + pOperator->pTaskInfo = pTaskInfo; + pOperator->exec = doSortedMerge; + pOperator->cleanupFn = destroySortedMergeOperatorInfo; - int64_t el = taosGetTimestampUs() - st; - pInfo->totalElapsed += el; + code = appendDownstream(pOperator, downstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - SDiskbasedBufStatis statis = getDBufStatis(pInfo->pSortInternalBuf); + return pOperator; - qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", GET_TASKID(pTaskInfo), t + 1, el, statis.loadBytes/1024.0, - statis.flushBytes/1024.0); + _error: + if (pInfo != NULL) { + destroySortedMergeOperatorInfo(pInfo, numOfOutput); } - pInfo->cmpParam.numOfSources = taosArrayGetSize(pInfo->pSources); - return 0; + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } static SSDataBlock* doSort(void* param, bool* newgroup) { @@ -6103,103 +6027,33 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SOrderOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pBlock = NULL; - if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); + return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes); } - int64_t st = taosGetTimestampUs(); + SSchema* p = blockDataExtractSchema(pInfo->pDataBlock, NULL); + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, + numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)"); - while(1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - // start to flush data into disk and try do multiway merge sort - if (pBlock == NULL) { - break; - } - - int32_t code = blockDataMerge(pInfo->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pTaskInfo->env, code); - } - - size_t size = blockDataGetSize(pInfo->pDataBlock); - if (size > pInfo->sortBufSize) { - // Perform the in-memory sort and then flush data in the buffer into disk. - int64_t p = taosGetTimestampUs(); - blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); - - int64_t el = taosGetTimestampUs() - p; - pInfo->sortElapsed += el; - - addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); - } - } - - if (pInfo->pDataBlock->info.rows > 0) { - // Perform the in-memory sort and then flush data in the buffer into disk. - blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); - - // All sorted data are resident in memory, external memory sort is not needed. - // Return to the upstream operator directly - if (isAllDataInMemBuf(pInfo->pSortInternalBuf)) { - pOperator->status = OP_EXEC_DONE; - return (pInfo->pDataBlock->info.rows == 0)? NULL:pInfo->pDataBlock; - } - - addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); - } - - doInternalSort(pTaskInfo, pInfo); - - int32_t code = blockDataEnsureCapacity(pInfo->pDataBlock, pInfo->numOfRowsInRes); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + tfree(p); + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - int32_t numOfSources = taosArrayGetSize(pInfo->pSources); - ASSERT(numOfSources <= getNumOfInMemBufPages(pInfo->pSortInternalBuf)); - code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, 0, numOfSources - 1, pInfo->pSortInternalBuf); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + ps->param = pOperator; + tsortAddSource(pInfo->pSortHandle, ps); - code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); + // TODO set error code; + int32_t code = tsortOpen(pInfo->pSortHandle); if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); + longjmp(pTaskInfo->env, terrno); } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); -} - -static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { - SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - - size_t numOfOrder = taosArrayGetSize(pOrderVal); - for (int32_t j = 0; j < numOfOrder; ++j) { - SBlockOrderInfo orderInfo = {0}; - SOrder* pOrder = taosArrayGet(pOrderVal, j); - orderInfo.order = pOrder->order; - - for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGet(pExprInfo, i); - if (pExpr->base.resSchema.colId == pOrder->col.info.colId) { - orderInfo.colIndex = i; - break; - } - } - - taosArrayPush(pOrderInfo, &orderInfo); - } - - return pOrderInfo; + return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes); } -SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal) { +SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6214,8 +6068,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pInfo->numOfRowsInRes = 1024; pInfo->pDataBlock = createOutputBuf_rv(pExprInfo, pInfo->numOfRowsInRes); - pInfo->pSources = taosArrayInit(4, POINTER_BYTES); - pInfo->cmpParam.orderInfo = createBlockOrder(pExprInfo, pOrderVal); + pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); @@ -6225,8 +6078,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI } } - int32_t code = createDiskbasedBuffer(&pInfo->pSortInternalBuf, pInfo->bufPageSize, pInfo->sortBufSize, 1, "/tmp/"); - if (pInfo->pSources == NULL || code != 0 || pInfo->cmpParam.orderInfo == NULL || pInfo->pDataBlock == NULL) { + if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { tfree(pOperator); destroyOrderOperatorInfo(pInfo, taosArrayGetSize(pExprInfo)); tfree(pInfo); @@ -6240,10 +6092,12 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; + + pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doSort; pOperator->cleanupFn = destroyOrderOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -6537,22 +6391,19 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { return NULL; } - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + STableIntervalOperatorInfo* pInfo = pOperator->info; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(pAggInfo->pGroupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes; + return pInfo->binfo.pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - +// int32_t order = pQueryAttr->order.order; +// STimeWindow win = pQueryAttr->window; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { @@ -6564,30 +6415,30 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { break; } -// setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; +// pQueryAttr->order.order = order; +// pQueryAttr->window = win; pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); + closeAllResultRows(&pInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + toSDatablock(&pInfo->groupResInfo, pInfo->pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; + return pInfo->binfo.pRes->info.rows == 0? NULL:pInfo->binfo.pRes; } static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { @@ -6602,11 +6453,11 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6627,8 +6478,8 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); + hashAllIntervalAgg(pOperator, &pIntervalInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value @@ -6636,18 +6487,18 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pQueryAttr->window = win; pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); + closeAllResultRows(&pIntervalInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + finalizeQueryResult(pOperator, pIntervalInfo->binfo.pCtx, &pIntervalInfo->binfo.resultRowInfo, pIntervalInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->binfo.resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes->info.rows == 0? NULL:pIntervalInfo->binfo.pRes; } static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { @@ -6663,14 +6514,14 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { int64_t st = taosGetTimestampUs(); // copyToSDataBlock(NULL, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } SQInfo* pQInfo = pRuntimeEnv->qinfo; pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6691,7 +6542,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); @@ -6703,11 +6554,11 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { @@ -6721,11 +6572,11 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6746,7 +6597,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); @@ -6759,14 +6610,14 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { int64_t st = taosGetTimestampUs(); // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } SQInfo* pQInfo = pRuntimeEnv->qinfo; pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { @@ -6818,8 +6669,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); pInfo->curWindow.skey = tsList[j]; pInfo->curWindow.ekey = tsList[j]; @@ -6840,8 +6691,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); } static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { @@ -7160,18 +7011,37 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } +static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + + pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); + pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + pAggSup->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); + + if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || + pAggSup->pResultRowHashTable == NULL || pAggSup->pool == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static void clearupAggSup(SAggSupporter* pAggSup) { + tfree(pAggSup->keyBuf); + taosHashCleanup(pAggSup->pResultRowHashTable); + taosHashCleanup(pAggSup->pResultRowListSet); + taosArrayDestroy(pAggSup->pResultRowArrayList); + destroyResultRowPool(pAggSup->pool); +} + static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t numOfRows, const STableGroupInfo* pTableGroupInfo) { pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); pInfo->binfo.capacity = 4096; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - - pInfo->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pInfo->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); - pInfo->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); - + initAggSup(&pInfo->aggSup, pExprInfo); pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); int32_t index = 0; @@ -7193,32 +7063,6 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n return TSDB_CODE_SUCCESS; } -static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); - - *dst = *src; - - dst->pExpr = exprdup(src->pExpr); - dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); - memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); - - memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); - for (int32_t j = 0; j < src->base.numOfParams; ++j) { - taosVariantAssign(&dst->base.param[j], &src->base.param[j]); - } -} - -static SExprInfo* exprArrayDup(SArray* pExprInfo) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); - for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - assignExprInfo(&p[i], pExpr); - } - - return p; -} - SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); @@ -7226,7 +7070,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE //(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); - setDefaultOutputBuf_rv(pInfo, MAIN_SCAN, pTaskInfo); + setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -7240,7 +7084,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; pOperator->cleanupFn = destroyAggOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7300,10 +7144,7 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); - taosArrayDestroy(pInfo->cmpParam.orderInfo); - destroyResultBuf(pInfo->pSortInternalBuf); - - tMergeTreeDestroy(pInfo->pMergeTree); + taosArrayDestroy(pInfo->orderInfo); } static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { @@ -7327,7 +7168,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; @@ -7340,7 +7181,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray pOperator->exec = doMultiTableAggregate; pOperator->cleanupFn = destroyAggOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7355,7 +7196,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pBInfo->resultRowInfo, 8); setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -7370,7 +7211,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doProjectOperation; pOperator->cleanupFn = destroyProjectOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7428,7 +7269,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanupFn = destroyConditionOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7446,32 +7287,49 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initAggSup(&pInfo->aggSup, pExprInfo); + + // todo: + pInfo->order = TSDB_ORDER_ASC; + pInfo->precision = TSDB_TIME_PRECISION_MICRO; + pInfo->win.skey = INT64_MIN; + pInfo->win.ekey = INT64_MAX; + pInfo->interval.intervalUnit = 's'; + pInfo->interval.slidingUnit = 's'; + pInfo->interval.interval = 1000; + pInfo->interval.sliding = 1000; + + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/"); + + int32_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TimeIntervalAggOperator"; -// pOperator->operatorType = OP_TimeWindow; + // pOperator->operatorType = OP_TimeWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; + pOperator->pExpr = exprArrayDup(pExprInfo); + + pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7479,9 +7337,9 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -7496,7 +7354,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->exec = doAllIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7506,7 +7364,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pInfo->reptScan = false; pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; @@ -7520,7 +7378,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->exec = doStateWindowAgg; pOperator->cleanupFn = destroyStateWindowOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { @@ -7528,7 +7386,7 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pInfo->prevTs = INT64_MIN; pInfo->reptScan = false; @@ -7545,16 +7403,16 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doSessionWindowAgg; pOperator->cleanupFn = destroySWindowOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; @@ -7569,16 +7427,16 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->exec = doSTableIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; @@ -7593,12 +7451,11 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->exec = doAllSTableIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } - SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index @@ -7612,7 +7469,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "GroupbyAggOperator"; @@ -7626,7 +7483,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = hashGroupbyAggregate; pOperator->cleanupFn = destroyGroupbyOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7642,7 +7499,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, @@ -7665,7 +7522,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->exec = doFill; pOperator->cleanupFn = destroySFillOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7713,7 +7570,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanupFn = destroySlimitOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -8011,7 +7868,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pExpr = pExpr; pOperator->cleanupFn = destroyDistinctOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -8201,16 +8058,22 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa STableGroupInfo groupInfo = {0}; int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + SArray* idList = NULL; + + if (groupInfo.numOfTables > 0) { + SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); + ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); - SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); - ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + // Transfer the Array of STableKeyInfo into uid list. + size_t numOfTables = taosArrayGetSize(pa); + idList = taosArrayInit(numOfTables, sizeof(uint64_t)); - // Transfer the Array of STableKeyInfo into uid list. - size_t numOfTables = taosArrayGetSize(pa); - SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); - for(int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); - taosArrayPush(idList, &pkeyInfo->uid); + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); + taosArrayPush(idList, &pkeyInfo->uid); + } + } else { + idList = taosArrayInit(4, sizeof(uint64_t)); } // SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pScanPhyNode->pScanCols, idList, pTaskInfo); @@ -8385,7 +8248,6 @@ int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { return TSDB_CODE_SUCCESS; } - static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { for (int32_t i = 0; i < numOfOutput; ++i) { int16_t functId = getExprFunctionId(&pExprs[i]); @@ -8549,66 +8411,6 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // return ret; } - -// todo refactor -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo *pUdfInfo) { -// *pExprInfo = NULL; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); -// if (pExprs == NULL) { -// return TSDB_CODE_QRY_OUT_OF_MEMORY; -// } -// -// bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); -// -// for (int32_t i = 0; i < numOfOutput; ++i) { -// pExprs[i].base = *pExpr[i]; -// memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); -// -// for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { -// taosVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); -// } -// -// pExprs[i].base.resSchema.type = 0; -// -// int16_t type = 0; -// int16_t bytes = 0; -// -// // parse the arithmetic expression -// if (pExprs[i].base.functionId == FUNCTION_ARITHM) { -// code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); -// -// if (code != TSDB_CODE_SUCCESS) { -// tfree(pExprs); -// return code; -// } -// -// type = TSDB_DATA_TYPE_DOUBLE; -// bytes = tDataTypes[type].bytes; -// } else { -// int32_t index = pExprs[i].base.colInfo.colIndex; -// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.pColumns->info.colId); -// -// type = prevExpr[index].base.resSchema.type; -// bytes = prevExpr[index].base.resSchema.bytes; -// } -// -// int32_t param = (int32_t)pExprs[i].base.param[0].i; -// if (getResultDataInfo(type, bytes, functionId, param, &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, -// &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { -// tfree(pExprs); -// return TSDB_CODE_QRY_INVALID_MSG; -// } -// -// assert(isValidDataType(pExprs[i].base.resSchema.type)); -// } -// -// *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; @@ -9017,30 +8819,3 @@ void releaseQueryBuf(size_t numOfTables) { // restore value is not enough buffer available atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } - -void freeQueryAttr(STaskAttr* pQueryAttr) { - if (pQueryAttr != NULL) { - if (pQueryAttr->fillVal != NULL) { - tfree(pQueryAttr->fillVal); - } - - pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); - - pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); - - tfree(pQueryAttr->tagColList); - tfree(pQueryAttr->pFilterInfo); - - pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); - - if (pQueryAttr->pGroupbyExpr != NULL) { - taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); - tfree(pQueryAttr->pGroupbyExpr); - } - -// filterFreeInfo(pQueryAttr->pFilters); - } -} - diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c new file mode 100644 index 0000000000000000000000000000000000000000..7f3da9bcac025c52337dec4f4348d17faf7b0dea --- /dev/null +++ b/source/libs/executor/src/tlinearhash.c @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tlinearhash.h" +#include "tdef.h" +#include "taoserror.h" +#include "tpagedbuf.h" + +#define LHASH_CAP_RATIO 0.85 + +// Always located in memory +typedef struct SLHashBucket { + SArray *pPageIdList; + int32_t size; // the number of element in this entry +} SLHashBucket; + +typedef struct SLHashObj { + SDiskbasedBuf *pBuf; + _hash_fn_t hashFn; + int32_t tuplesPerPage; + SLHashBucket **pBucket; // entry list + int32_t numOfAlloc; // number of allocated bucket ptr slot + int32_t bits; // the number of bits used in hash + int32_t numOfBuckets; // the number of buckets + int64_t size; // the number of total items +} SLHashObj; + +/** + * the data struct for each hash node + * +-----------+-------+--------+ + * | SLHashNode| key | data | + * +-----------+-------+--------+ + */ +typedef struct SLHashNode { + uint16_t keyLen; + uint16_t dataLen; +} SLHashNode; + +#define GET_LHASH_NODE_KEY(_n) (((char*)(_n)) + sizeof(SLHashNode)) +#define GET_LHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen) +#define GET_LHASH_NODE_LEN(_n) (sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen + ((SLHashNode*)(_n))->dataLen) + +static int32_t doAddNewBucket(SLHashObj* pHashObj); + +static int32_t doGetBucketIdFromHashVal(int32_t hashv, int32_t bits) { + return hashv & ((1ul << (bits)) - 1); +} + +static int32_t doGetAlternativeBucketId(int32_t bucketId, int32_t bits, int32_t numOfBuckets) { + int32_t v = bucketId - (1ul << (bits - 1)); + ASSERT(v < numOfBuckets); + return v; +} + +static int32_t doGetRelatedSplitBucketId(int32_t bucketId, int32_t bits) { + int32_t splitBucketId = (1ul << (bits - 1)) ^ bucketId; + return splitBucketId; +} + +static void doCopyObject(char* p, const void* key, int32_t keyLen, const void* data, int32_t size) { + *(uint16_t*) p = keyLen; + p += sizeof(uint16_t); + *(uint16_t*) p = size; + p += sizeof(uint16_t); + + memcpy(p, key, keyLen); + p += keyLen; + + memcpy(p, data, size); +} + +static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t index, const void* key, int32_t keyLen, + const void* data, int32_t size) { + int32_t pageId = *(int32_t*)taosArrayGetLast(pBucket->pPageIdList); + + SFilePage* pPage = getBufPage(pHashObj->pBuf, pageId); + ASSERT (pPage != NULL); + + // put to current buf page + size_t nodeSize = sizeof(SLHashNode) + keyLen + size; + ASSERT(nodeSize + sizeof(SFilePage) <= getBufPageSize(pHashObj->pBuf)); + + if (pPage->num + nodeSize > getBufPageSize(pHashObj->pBuf)) { + releaseBufPage(pHashObj->pBuf, pPage); + + // allocate the overflow buffer page to hold this k/v. + int32_t newPageId = -1; + SFilePage* pNewPage = getNewBufPage(pHashObj->pBuf, 0, &newPageId); + if (pNewPage == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBucket->pPageIdList, &newPageId); + + doCopyObject(pNewPage->data, key, keyLen, data, size); + pNewPage->num = sizeof(SFilePage) + nodeSize; + + setBufPageDirty(pNewPage, true); + releaseBufPage(pHashObj->pBuf, pNewPage); + } else { + char* p = (char*) pPage + pPage->num; + doCopyObject(p, key, keyLen, data, size); + pPage->num += nodeSize; + setBufPageDirty(pPage, true); + releaseBufPage(pHashObj->pBuf, pPage); + } + + pBucket->size += 1; +// printf("===> add to bucket:0x%x, num:%d, key:%d\n", index, pBucket->size, *(int*) key); + + return TSDB_CODE_SUCCESS; +} + +static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket* pBucket) { + ASSERT(pPage != NULL && pNode != NULL && pBucket->size >= 1); + + int32_t len = GET_LHASH_NODE_LEN(pNode); + char* p = (char*) pNode + len; + + char* pEnd = (char*)pPage + pPage->num; + memmove(pNode, p, (pEnd - p)); + + pPage->num -= len; + if (pPage->num == 0) { + // this page is empty, could be recycle in the future. + } + + setBufPageDirty(pPage, true); + + pBucket->size -= 1; +} + +static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { + size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); + if (numOfPages <= 1) { + return; + } + + int32_t* firstPage = taosArrayGet(pBucket->pPageIdList, 0); + SFilePage* pFirst = getBufPage(pHashObj->pBuf, *firstPage); + + int32_t* pageId = taosArrayGetLast(pBucket->pPageIdList); + SFilePage* pLast = getBufPage(pHashObj->pBuf, *pageId); + + if (pLast->num <= sizeof(SFilePage)) { + // this is empty + dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); + return; + } + + char* pStart = pLast->data; + int32_t nodeSize = GET_LHASH_NODE_LEN(pStart); + while (1) { + if (pFirst->num + nodeSize < getBufPageSize(pHashObj->pBuf)) { + char* p = ((char*)pFirst) + pFirst->num; + + SLHashNode* pNode = (SLHashNode*)pStart; + doCopyObject(p, GET_LHASH_NODE_KEY(pStart), pNode->keyLen, GET_LHASH_NODE_DATA(pStart), pNode->dataLen); + + setBufPageDirty(pFirst, true); + setBufPageDirty(pLast, true); + + ASSERT(pLast->num >= nodeSize + sizeof(SFilePage)); + + pFirst->num += nodeSize; + pLast->num -= nodeSize; + + pStart += nodeSize; + if (pLast->num <= sizeof(SFilePage)) { + // this is empty + dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); + break; + } + + nodeSize = GET_LHASH_NODE_LEN(pStart); + } else { // move to the front of pLast page + if (pStart != pLast->data) { + memmove(pLast->data, pStart, (((char*)pLast) + pLast->num - pStart)); + setBufPageDirty(pLast, true); + } + + releaseBufPage(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + break; + } + } +} + +static int32_t doAddNewBucket(SLHashObj* pHashObj) { + if (pHashObj->numOfBuckets + 1 > pHashObj->numOfAlloc) { + int32_t newLen = pHashObj->numOfAlloc * 1.25; + if (newLen == pHashObj->numOfAlloc) { + newLen += 4; + } + + char* p = realloc(pHashObj->pBucket, POINTER_BYTES * newLen); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memset(p + POINTER_BYTES * pHashObj->numOfBuckets, 0, newLen - pHashObj->numOfBuckets); + pHashObj->pBucket = (SLHashBucket**) p; + pHashObj->numOfAlloc = newLen; + } + + SLHashBucket* pBucket = calloc(1, sizeof(SLHashBucket)); + pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; + + pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); + if (pBucket->pPageIdList == NULL || pBucket == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t pageId = -1; + SFilePage* p = getNewBufPage(pHashObj->pBuf, 0, &pageId); + p->num = sizeof(SFilePage); + setBufPageDirty(p, true); + + releaseBufPage(pHashObj->pBuf, p); + taosArrayPush(pBucket->pPageIdList, &pageId); + + pHashObj->numOfBuckets += 1; +// printf("---------------add new bucket, id:0x%x, total:%d\n", pHashObj->numOfBuckets - 1, pHashObj->numOfBuckets); + return TSDB_CODE_SUCCESS; +} + +SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage) { + SLHashObj* pHashObj = calloc(1, sizeof(SLHashObj)); + if (pHashObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, "/tmp"); + if (code != 0) { + terrno = code; + return NULL; + } + + setBufPageCompressOnDisk(pHashObj->pBuf, false); + + /** + * The number of bits in the hash value, which is used to decide the exact bucket where the object should be located in. + * The initial value is 0. + */ + pHashObj->bits = 0; + pHashObj->hashFn = fn; + pHashObj->tuplesPerPage = numOfTuplePerPage; + + pHashObj->numOfAlloc = 4; // initial allocated array list + pHashObj->pBucket = calloc(pHashObj->numOfAlloc, POINTER_BYTES); + + code = doAddNewBucket(pHashObj); + if (code != TSDB_CODE_SUCCESS) { + destroyDiskbasedBuf(pHashObj->pBuf); + tfree(pHashObj); + terrno = code; + return NULL; + } + + return pHashObj; +} + +void* tHashCleanup(SLHashObj* pHashObj) { + destroyDiskbasedBuf(pHashObj->pBuf); + for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { + taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList); + tfree(pHashObj->pBucket[i]); + } + + tfree(pHashObj->pBucket); + tfree(pHashObj); + return NULL; +} + +int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size) { + ASSERT(pHashObj != NULL && key != NULL); + + if (pHashObj->bits == 0) { + SLHashBucket* pBucket = pHashObj->pBucket[0]; + doAddToBucket(pHashObj, pBucket, 0, key, keyLen, data, size); + } else { + int32_t hashVal = pHashObj->hashFn(key, keyLen); + int32_t v = doGetBucketIdFromHashVal(hashVal, pHashObj->bits); + + if (v >= pHashObj->numOfBuckets) { + int32_t newBucketId = doGetAlternativeBucketId(v, pHashObj->bits, pHashObj->numOfBuckets); +// printf("bucketId: 0x%x not exists, put it into 0x%x instead\n", v, newBucketId); + v = newBucketId; + } + + SLHashBucket* pBucket = pHashObj->pBucket[v]; + int32_t code = doAddToBucket(pHashObj, pBucket, v, key, keyLen, data, size); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pHashObj->size += 1; + + // Too many records, needs to bucket split + if ((pHashObj->numOfBuckets * LHASH_CAP_RATIO * pHashObj->tuplesPerPage) < pHashObj->size) { + int32_t newBucketId = pHashObj->numOfBuckets; + + int32_t code = doAddNewBucket(pHashObj); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t numOfBits = ceil(log(pHashObj->numOfBuckets) / log(2)); + if (numOfBits > pHashObj->bits) { +// printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId); + ASSERT(numOfBits == pHashObj->bits + 1); + pHashObj->bits = numOfBits; + } + + int32_t splitBucketId = doGetRelatedSplitBucketId(newBucketId, pHashObj->bits); + + // load all data in this bucket and check if the data needs to relocated into the new bucket + SLHashBucket* pBucket = pHashObj->pBucket[splitBucketId]; +// printf("split %d items' bucket:0x%x to new bucket:0x%x\n", pBucket->size, splitBucketId, newBucketId); + + for (int32_t i = 0; i < taosArrayGetSize(pBucket->pPageIdList); ++i) { + int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i); + SFilePage* p = getBufPage(pHashObj->pBuf, pageId); + + char* pStart = p->data; + while (pStart - ((char*) p) < p->num) { + SLHashNode* pNode = (SLHashNode*)pStart; + ASSERT(pNode->keyLen > 0 && pNode->dataLen >= 0); + + char* k = GET_LHASH_NODE_KEY(pNode); + int32_t hashv = pHashObj->hashFn(k, pNode->keyLen); + int32_t v1 = doGetBucketIdFromHashVal(hashv, pHashObj->bits); + + if (v1 != splitBucketId) { // place it into the new bucket + ASSERT(v1 == newBucketId); +// printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1); + + SLHashBucket* pNewBucket = pHashObj->pBucket[newBucketId]; + doAddToBucket(pHashObj, pNewBucket, newBucketId, (void*)GET_LHASH_NODE_KEY(pNode), pNode->keyLen, + GET_LHASH_NODE_KEY(pNode), pNode->dataLen); + doRemoveFromBucket(p, pNode, pBucket); + } else { +// printf("check key:%d, located into: %d, skip it\n", *(int*) k, v1); + + int32_t nodeSize = GET_LHASH_NODE_LEN(pStart); + pStart += nodeSize; + } + } + releaseBufPage(pHashObj->pBuf, p); + } + + doCompressBucketPages(pHashObj, pBucket); + } + + return TSDB_CODE_SUCCESS; +} + +char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) { + ASSERT(pHashObj != NULL && key != NULL && keyLen > 0); + int32_t hashv = pHashObj->hashFn(key, keyLen); + + int32_t bucketId = doGetBucketIdFromHashVal(hashv, pHashObj->bits); + if (bucketId >= pHashObj->numOfBuckets) { + bucketId = doGetAlternativeBucketId(bucketId, pHashObj->bits, pHashObj->numOfBuckets); + } + + SLHashBucket* pBucket = pHashObj->pBucket[bucketId]; + for (int32_t i = 0; i < taosArrayGetSize(pBucket->pPageIdList); ++i) { + int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i); + SFilePage* p = getBufPage(pHashObj->pBuf, pageId); + + char* pStart = p->data; + while (pStart - p->data < p->num) { + SLHashNode* pNode = (SLHashNode*)pStart; + + char* k = GET_LHASH_NODE_KEY(pNode); + if (pNode->keyLen == keyLen && (memcmp(key, k, keyLen) == 0)) { + releaseBufPage(pHashObj->pBuf, p); + return GET_LHASH_NODE_DATA(pNode); + } else { + pStart += GET_LHASH_NODE_LEN(pStart); + } + } + + releaseBufPage(pHashObj->pBuf, p); + } + + return NULL; +} + +int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen) { + // todo +} + +void tHashPrint(const SLHashObj* pHashObj, int32_t type) { + printf("==================== linear hash ====================\n"); + printf("total bucket:%d, size:%ld, ratio:%.2f\n", pHashObj->numOfBuckets, pHashObj->size, LHASH_CAP_RATIO); + + dBufSetPrintInfo(pHashObj->pBuf); + + if (type == LINEAR_HASH_DATA) { + for (int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { +// printf("bucket: 0x%x, obj:%d, page:%d\n", i, pHashObj->pBucket[i]->size, +// (int)taosArrayGetSize(pHashObj->pBucket[i]->pPageIdList)); + } + } else { + dBufPrintStatis(pHashObj->pBuf); + } +} \ No newline at end of file diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c new file mode 100644 index 0000000000000000000000000000000000000000..d042dc0effc536ac9124e9f2c2688e4c5715ab54 --- /dev/null +++ b/source/libs/executor/src/tsort.c @@ -0,0 +1,679 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tcommon.h" +#include "query.h" + +#include "tsort.h" +#include "tep.h" +#include "tdef.h" +#include "tlosertree.h" +#include "tpagedbuf.h" +#include "tutil.h" + +typedef struct STupleHandle { + SSDataBlock* pBlock; + int32_t rowIndex; +} STupleHandle; + +typedef struct SSortHandle { + int32_t type; + + int32_t pageSize; + int32_t numOfPages; + SDiskbasedBuf *pBuf; + + SArray *pOrderInfo; + bool nullFirst; + bool hasVarCol; + SArray *pOrderedSource; + + _sort_fetch_block_fn_t fetchfp; + _sort_merge_compar_fn_t comparFn; + + void *pParam; + SMultiwayMergeTreeInfo *pMergeTree; + int32_t numOfCols; + + int64_t startTs; + uint64_t sortElapsed; + uint64_t totalElapsed; + + int32_t sourceId; + SSDataBlock *pDataBlock; + SMsortComparParam cmpParam; + int32_t numOfCompletedSources; + bool opened; + const char *idStr; + + bool inMemSort; + bool needAdjust; + STupleHandle tupleHandle; +} SSortHandle; + +static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); + +static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { + SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + + colInfo.info.type = pSchema[i].type; + colInfo.info.bytes = pSchema[i].bytes; + colInfo.info.colId = pSchema[i].colId; + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + return pBlock; +} + +/** + * + * @param type + * @return + */ +SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) { + SSortHandle* pSortHandle = calloc(1, sizeof(SSortHandle)); + + pSortHandle->type = type; + pSortHandle->pageSize = pageSize; + pSortHandle->numOfPages = numOfPages; + pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); + pSortHandle->pOrderInfo = pOrderInfo; + pSortHandle->nullFirst = nullFirst; + pSortHandle->cmpParam.orderInfo = pOrderInfo; + + pSortHandle->pDataBlock = createDataBlock_rv(pSchema, numOfCols); + tsortSetComparFp(pSortHandle, msortComparFn); + + if (idstr != NULL) { + pSortHandle->idStr = strdup(idstr); + } + + return pSortHandle; +} + +void tsortDestroySortHandle(SSortHandle* pSortHandle) { + tsortClose(pSortHandle); + if (pSortHandle->pMergeTree != NULL) { + tMergeTreeDestroy(pSortHandle->pMergeTree); + } + + destroyDiskbasedBuf(pSortHandle->pBuf); + tfree(pSortHandle->idStr); + tfree(pSortHandle); +} + +int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { + taosArrayPush(pSortHandle->pOrderedSource, &pSource); +} + +static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { + SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + if (pSource == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + pSource->pageIdList = getDataBufPagesIdList(pBuf, (*sourceId)); + pSource->src.pBlock = pBlock; + + taosArrayPush(pAllSources, &pSource); + + (*sourceId) += 1; + + int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize; + + return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); +} + +static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { + int32_t start = 0; + + if (pHandle->pBuf == NULL) { + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + while(start < pDataBlock->info.rows) { + int32_t stop = 0; + blockDataSplitRows(pDataBlock, pHandle->hasVarCol, start, &stop, pHandle->pageSize); + SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1); + if (p == NULL) { + return terrno; + } + + int32_t pageId = -1; + SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + if (pPage == NULL) { + return terrno; + } + + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage->data, p); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + blockDataDestroy(p); + start = stop + 1; + } + + blockDataClearup(pDataBlock, pHandle->hasVarCol); + + SSDataBlock* pBlock = createOneDataBlock(pDataBlock); + int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } +} + +static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { + cmpParam->pSources = taosArrayGet(pSources, startIndex); + cmpParam->numOfSources = (endIndex - startIndex + 1); + + int32_t code = 0; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pHandle->pBuf, pPage); + } + } else { + // multi-pass internal merge sort is required + if (pHandle->pBuf == NULL) { + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SGenericSource* pSource = cmpParam->pSources[i]; + pSource->src.pBlock = pHandle->fetchfp(pSource->param); + } + } + + return code; +} + +static int32_t sortComparClearup(SMsortComparParam* cmpParam) { + for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + blockDataDestroy(pSource->src.pBlock); + tfree(pSource); + } + + cmpParam->numOfSources = 0; +} + +static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); + bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); + + if (isNull) { + colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + } else { + char* pData = colDataGetData(pSrcColInfo, *rowIndex); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } + } + + pBlock->info.rows += 1; + *rowIndex += 1; +} + +static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SSortHandle *pHandle, int32_t* numOfCompleted) { + /* + * load a new SDataBlock into memory of a given intermediate data-set source, + * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree + */ + if (pSource->src.rowIndex >= pSource->src.pBlock->info.rows) { + pSource->src.rowIndex = 0; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { + (*numOfCompleted) += 1; + pSource->src.rowIndex = -1; + pSource->pageIndex = -1; + pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); + } else { + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pHandle->pBuf, pPage); + } + } else { + pSource->src.pBlock = pHandle->fetchfp(((SGenericSource*)pSource)->param); + if (pSource->src.pBlock == NULL) { + (*numOfCompleted) += 1; + pSource->src.rowIndex = -1; + } + } + } + + /* + * Adjust loser tree otherwise, according to new candidate data + * if the loser tree is rebuild completed, we do not need to adjust + */ + int32_t leafNodeIndex = tMergeTreeGetAdjustIndex(pTree); + +#ifdef _DEBUG_VIEW + printf("before adjust:\t"); + tMergeTreePrint(pTree); +#endif + + tMergeTreeAdjust(pTree, leafNodeIndex); + +#ifdef _DEBUG_VIEW + printf("\nafter adjust:\t"); + tMergeTreePrint(pTree); +#endif +} + +static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { + blockDataClearup(pHandle->pDataBlock, pHandle->hasVarCol); + + while(1) { + if (cmpParam->numOfSources == pHandle->numOfCompletedSources) { + break; + } + + int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + + SExternalMemSource *pSource = (*cmpParam).pSources[index]; + appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex); + + int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + if (pHandle->pDataBlock->info.rows >= capacity) { + return pHandle->pDataBlock; + } + } + + return (pHandle->pDataBlock->info.rows > 0)? pHandle->pDataBlock:NULL; +} + +int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t pLeftIdx = *(int32_t *)pLeft; + int32_t pRightIdx = *(int32_t *)pRight; + + SMsortComparParam *pParam = (SMsortComparParam *)param; + + SArray *pInfo = pParam->orderInfo; + + SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; + SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; + + // this input is exhausted, set the special value to denote this + if (pLeftSource->src.rowIndex == -1) { + return 1; + } + + if (pRightSource->src.rowIndex == -1) { + return -1; + } + + SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; + SSDataBlock* pRightBlock = pRightSource->src.pBlock; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + } + + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pParam->nullFirst? 1:-1; + } + + if (leftNull) { + return pParam->nullFirst? -1:1; + } + + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + + switch(pLeftColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftv = *(int32_t*)left1; + int32_t rightv = *(int32_t*)right1; + + if (leftv == rightv) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return leftv < rightv? -1 : 1; + } else { + return leftv < rightv? 1 : -1; + } + } + } + default: + assert(0); + } + } +} + +static int32_t doInternalMergeSort(SSortHandle* pHandle) { + size_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + + // Calculate the I/O counts to complete the data sort. + double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages)); + + pHandle->totalElapsed = taosGetTimestampUs() - pHandle->startTs; + qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, + pHandle->idStr, (int32_t) (sortPass + 1), getTotalBufSize(pHandle->pBuf), pHandle->sortElapsed, pHandle->totalElapsed); + + size_t pgSize = pHandle->pageSize; + int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pHandle->pDataBlock))/ blockDataGetSerialRowSize(pHandle->pDataBlock); + + blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); + + size_t numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); + for(int32_t t = 0; t < sortPass; ++t) { + int64_t st = taosGetTimestampUs(); + + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + + int32_t numOfInputSources = pHandle->numOfPages; + int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources; + + // Only *numOfInputSources* can be loaded into buffer to perform the external sort. + for(int32_t i = 0; i < sortGroup; ++i) { + pHandle->sourceId += 1; + + int32_t end = (i + 1) * numOfInputSources - 1; + if (end > numOfSorted - 1) { + end = numOfSorted - 1; + } + + pHandle->cmpParam.numOfSources = end - i * numOfInputSources + 1; + + int32_t code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, i * numOfInputSources, end, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + while (1) { + SSDataBlock* pDataBlock = getSortedBlockData(pHandle, &pHandle->cmpParam, numOfRows); + if (pDataBlock == NULL) { + break; + } + + int32_t pageId = -1; + SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + if (pPage == NULL) { + return terrno; + } + + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage->data, pDataBlock); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + blockDataClearup(pDataBlock, pHandle->hasVarCol); + } + + tMergeTreeDestroy(pHandle->pMergeTree); + pHandle->numOfCompletedSources = 0; + + SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock); + code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId); + if (code != 0) { + return code; + } + } + + sortComparClearup(&pHandle->cmpParam); + + taosArrayClear(pHandle->pOrderedSource); + taosArrayAddAll(pHandle->pOrderedSource, pResList); + taosArrayDestroy(pResList); + + numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); + + int64_t el = taosGetTimestampUs() - st; + pHandle->totalElapsed += el; + + SDiskbasedBufStatis statis = getDBufStatis(pHandle->pBuf); + qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", pHandle->idStr, t + 1, el, statis.loadBytes/1024.0, + statis.flushBytes/1024.0); + + if (pHandle->type == SORT_MULTISOURCE_MERGE) { + pHandle->type = SORT_SINGLESOURCE_SORT; + pHandle->comparFn = msortComparFn; + } + } + + pHandle->cmpParam.numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + return 0; +} + +static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { + size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + SGenericSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); + taosArrayClear(pHandle->pOrderedSource); + + while (1) { + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (pBlock == NULL) { + break; + } + + if (pHandle->pDataBlock == NULL) { + pHandle->pDataBlock = createOneDataBlock(pBlock); + } + + int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); + if (code != 0) { + return code; + } + + size_t size = blockDataGetSize(pHandle->pDataBlock); + if (size > sortBufSize) { + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst); + + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + doAddToBuf(pHandle->pDataBlock, pHandle); + } + } + + if (pHandle->pDataBlock->info.rows > 0) { + size_t size = blockDataGetSize(pHandle->pDataBlock); + + // Perform the in-memory sort and then flush data in the buffer into disk. + blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst); + + // All sorted data can fit in memory, external memory sort is not needed. Return to directly + if (size <= sortBufSize) { + pHandle->cmpParam.numOfSources = 1; + pHandle->inMemSort = true; + + pHandle->tupleHandle.rowIndex = -1; + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; + return 0; + } else { + doAddToBuf(pHandle->pDataBlock, pHandle); + } + } + + tfree(source); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tsortOpen(SSortHandle* pHandle) { + if (pHandle->opened) { + return 0; + } + + if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { + return -1; + } + + pHandle->opened = true; + + int32_t code = createInitialSortedMultiSources(pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // do internal sort + code = doInternalMergeSort(pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + if (pHandle->pBuf != NULL) { + ASSERT(numOfSources <= getNumOfInMemBufPages(pHandle->pBuf)); + } + + code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, 0, numOfSources - 1, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + if (code != TSDB_CODE_SUCCESS) { + return code; + } +} + +int32_t tsortClose(SSortHandle* pHandle) { + // do nothing +} + +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp) { + pHandle->fetchfp = fp; +} + +int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { + pHandle->comparFn = fp; +} + +STupleHandle* tsortNextTuple(SSortHandle* pHandle) { + if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { + return NULL; + } + + // All the data are hold in the buffer, no external sort is invoked. + if (pHandle->inMemSort) { + pHandle->tupleHandle.rowIndex += 1; + if (pHandle->tupleHandle.rowIndex == pHandle->pDataBlock->info.rows) { + pHandle->numOfCompletedSources = 1; + return NULL; + } + + return &pHandle->tupleHandle; + } + + int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + SExternalMemSource *pSource = pHandle->cmpParam.pSources[index]; + + if (pHandle->needAdjust) { + int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + } + + if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { + return NULL; + } + + // Get the adjusted value after the loser tree is updated. + index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + pSource = pHandle->cmpParam.pSources[index]; + + assert(pSource->src.pBlock != NULL); + + pHandle->tupleHandle.rowIndex = pSource->src.rowIndex; + pHandle->tupleHandle.pBlock = pSource->src.pBlock; + + pHandle->needAdjust = true; + pSource->src.rowIndex += 1; + + return &pHandle->tupleHandle; +} + +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { + return false; +} + +void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { + SColumnInfoData* pColInfo = TARRAY_GET_ELEM(pVHandle->pBlock->pDataBlock, colIndex); + return colDataGetData(pColInfo, pVHandle->rowIndex); +} diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 6fcaf3536a4dd7dbbfdbc9cc76451bebc7aa0d9e..de64958793480b8e68bd6a9b1dc622faaa512f45 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #include #include @@ -37,22 +38,30 @@ namespace { +enum { + data_rand = 0x1, + data_asc = 0x2, + data_desc = 0x3, +}; + typedef struct SDummyInputInfo { - int32_t max; - int32_t current; - int32_t startVal; + int32_t totalPages; // numOfPages + int32_t current; + int32_t startVal; + int32_t type; + int32_t numOfRowsPerPage; + int32_t numOfCols; // number of columns + int64_t tsStart; SSDataBlock* pBlock; } SDummyInputInfo; SSDataBlock* getDummyBlock(void* param, bool* newgroup) { SOperatorInfo* pOperator = static_cast(param); SDummyInputInfo* pInfo = static_cast(pOperator->info); - if (pInfo->current >= pInfo->max) { + if (pInfo->current >= pInfo->totalPages) { return NULL; } - int32_t numOfRows = 1000; - if (pInfo->pBlock == NULL) { pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); @@ -62,8 +71,8 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(numOfRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (numOfRows + 7) / 8)); + colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -85,10 +94,18 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { char buf[128] = {0}; char b1[128] = {0}; - for(int32_t i = 0; i < numOfRows; ++i) { + int32_t v = 0; + for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); - int32_t v = (--pInfo->startVal); + if (pInfo->type == data_desc) { + v = (--pInfo->startVal); + } else if (pInfo->type == data_asc) { + v = ++pInfo->startVal; + } else if (pInfo->type == data_rand) { + v = random(); + } + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); // sprintf(buf, "this is %d row", i); @@ -98,21 +115,103 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { // colDataAppend(pColInfo2, i, b1, false); } - pBlock->info.rows = numOfRows; + pBlock->info.rows = pInfo->numOfRowsPerPage; pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; } -SOperatorInfo* createDummyOperator(int32_t numOfBlocks) { +SSDataBlock* get2ColsDummyBlock(void* param, bool* newgroup) { + SOperatorInfo* pOperator = static_cast(param); + SDummyInputInfo* pInfo = static_cast(pOperator->info); + if (pInfo->current >= pInfo->totalPages) { + return NULL; + } + + if (pInfo->pBlock == NULL) { + pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + + pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; + colInfo.info.bytes = sizeof(int64_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); +// colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + + SColumnInfoData colInfo1 = {0}; + colInfo1.info.type = TSDB_DATA_TYPE_INT; + colInfo1.info.bytes = 4; + colInfo1.info.colId = 2; + + colInfo1.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo1.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + } else { + blockDataClearup(pInfo->pBlock, false); + } + + SSDataBlock* pBlock = pInfo->pBlock; + + char buf[128] = {0}; + char b1[128] = {0}; + int64_t ts = 0; + int32_t v = 0; + for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) { + SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); + + ts = (++pInfo->tsStart); + colDataAppend(pColInfo, i, reinterpret_cast(&ts), false); + + SColumnInfoData* pColInfo1 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); + if (pInfo->type == data_desc) { + v = (--pInfo->startVal); + } else if (pInfo->type == data_asc) { + v = ++pInfo->startVal; + } else if (pInfo->type == data_rand) { + v = random(); + } + + colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); + +// sprintf(buf, "this is %d row", i); +// STR_TO_VARSTR(b1, buf); +// +// SColumnInfoData* pColInfo2 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); +// colDataAppend(pColInfo2, i, b1, false); + } + + pBlock->info.rows = pInfo->numOfRowsPerPage; + pBlock->info.numOfCols = 1; + + pInfo->current += 1; + + blockDataUpdateTsWindow(pBlock); + return pBlock; + +} + +SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) { SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); pOperator->name = "dummyInputOpertor4Test"; - pOperator->exec = getDummyBlock; + + if (numOfCols == 1) { + pOperator->exec = getDummyBlock; + } else { + pOperator->exec = get2ColsDummyBlock; + } SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); - pInfo->max = numOfBlocks; - pInfo->startVal = 1500000; + pInfo->totalPages = numOfBlocks; + pInfo->startVal = startVal; + pInfo->numOfRowsPerPage = rowsPerPage; + pInfo->type = type; + pInfo->tsStart = 1620000000000; pOperator->info = pInfo; return pOperator; @@ -722,6 +821,7 @@ TEST(testCase, build_executor_tree_Test) { } #if 0 + TEST(testCase, inMem_sort_Test) { SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); SOrder o = {.order = TSDB_ORDER_ASC}; @@ -738,7 +838,7 @@ TEST(testCase, inMem_sort_Test) { exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); taosArrayPush(pExprInfo, &exp1); - SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal); + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL); bool newgroup = false; SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup); @@ -746,7 +846,7 @@ TEST(testCase, inMem_sort_Test) { SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); for(int32_t i = 0; i < pRes->info.rows; ++i) { - char* p = colDataGet(pCol2, i); + char* p = colDataGetData(pCol2, i); printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); } } @@ -771,6 +871,7 @@ int32_t cmp(const void* p1, const void* p2) { } } +#if 0 TEST(testCase, external_sort_Test) { #if 0 su* v = static_cast(calloc(1000000, sizeof(su))); @@ -788,7 +889,6 @@ TEST(testCase, external_sort_Test) { return; #endif -#if 0 srand(time(NULL)); SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); @@ -807,7 +907,7 @@ TEST(testCase, external_sort_Test) { exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); - SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal); + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL); bool newgroup = false; SSDataBlock* pRes = NULL; @@ -833,13 +933,157 @@ TEST(testCase, external_sort_Test) { SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); // SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); for (int32_t i = 0; i < pRes->info.rows; ++i) { -// char* p = colDataGet(pCol2, i); +// char* p = colDataGetData(pCol2, i); printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]); // printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); } } - printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf); + int64_t s2 = taosGetTimestampUs(); + printf("total:%ld\n", s2 - s1); + + pOperator->cleanupFn(pOperator->info, 2); + tfree(exp); + tfree(exp1); + taosArrayDestroy(pExprInfo); + taosArrayDestroy(pOrderVal); +} + +TEST(testCase, sorted_merge_Test) { + srand(time(NULL)); + + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result"); + exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns->flag = TSDB_COL_NORMAL; + exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; + exp->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); +// taosArrayPush(pExprInfo, &exp1); + + int32_t numOfSources = 10; + SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); + for(int32_t i = 0; i < numOfSources; ++i) { + plist[i] = createDummyOperator(1, 1, 1, data_asc); + } + + SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL); + + bool newgroup = false; + SSDataBlock* pRes = NULL; + + int32_t total = 1; + + int64_t s1 = taosGetTimestampUs(); + int32_t t = 1; + + while(1) { + int64_t s = taosGetTimestampUs(); + pRes = pOperator->exec(pOperator, &newgroup); + + int64_t e = taosGetTimestampUs(); + if (t++ == 1) { + printf("---------------elapsed:%ld\n", e - s); + } + + if (pRes == NULL) { + break; + } + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for (int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGetData(pCol2, i); + printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); +// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } + } + + int64_t s2 = taosGetTimestampUs(); + printf("total:%ld\n", s2 - s1); + + pOperator->cleanupFn(pOperator->info, 2); + tfree(exp); + tfree(exp1); + taosArrayDestroy(pExprInfo); + taosArrayDestroy(pOrderVal); +} + +TEST(testCase, time_interval_Operator_Test) { + srand(time(NULL)); + + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, "ts"); + exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns->flag = TSDB_COL_NORMAL; + exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8}; + exp->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, 8, 2, "res1"); + exp1->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp1->base.pColumns->flag = TSDB_COL_NORMAL; + exp1->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; + exp1->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp1); + + SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2); + + SExecTaskInfo ti = {0}; + SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti); + + bool newgroup = false; + SSDataBlock* pRes = NULL; + + int32_t total = 1; + + int64_t s1 = taosGetTimestampUs(); + int32_t t = 1; + + while(1) { + int64_t s = taosGetTimestampUs(); + pRes = pOperator->exec(pOperator, &newgroup); + + int64_t e = taosGetTimestampUs(); + if (t++ == 1) { + printf("---------------elapsed:%ld\n", e - s); + } + + if (pRes == NULL) { + break; + } + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for (int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGetData(pCol2, i); + printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); +// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } + } int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); @@ -849,6 +1093,7 @@ TEST(testCase, external_sort_Test) { tfree(exp1); taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); -#endif } +#endif + #pragma GCC diagnostic pop diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..66ef3b0877d67f3967a1b642e03d84c4e979c8d5 --- /dev/null +++ b/source/libs/executor/test/lhashTests.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "executorimpl.h" +#include "tlinearhash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +TEST(testCase, linear_hash_Tests) { + srand(time(NULL)); + + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); +#if 0 + SLHashObj* pHashObj = tHashInit(256, 4096, fn, 320); + for(int32_t i = 0; i < 5000000; ++i) { + int32_t code = tHashPut(pHashObj, &i, sizeof(i), &i, sizeof(i)); + assert(code == 0); + } + +// tHashPrint(pHashObj, LINEAR_HASH_STATIS); + +// for(int32_t i = 0; i < 10000; ++i) { +// char* v = tHashGet(pHashObj, &i, sizeof(i)); +// if (v != NULL) { +//// printf("find value: %d, key:%d\n", *(int32_t*) v, i); +// } else { +// printf("failed to found key:%d in hash\n", i); +// } +// } + + tHashPrint(pHashObj, LINEAR_HASH_STATIS); + tHashCleanup(pHashObj); +#endif + +#if 0 + SHashObj* pHashObj = taosHashInit(1000, fn, false, HASH_NO_LOCK); + for(int32_t i = 0; i < 1000000; ++i) { + taosHashPut(pHashObj, &i, sizeof(i), &i, sizeof(i)); + } + + for(int32_t i = 0; i < 10000; ++i) { + void* v = taosHashGet(pHashObj, &i, sizeof(i)); + } + taosHashCleanup(pHashObj); +#endif + +} \ No newline at end of file diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fc366e4cc872ca365c36b8724e7cb40e0ee29a40 --- /dev/null +++ b/source/libs/executor/test/sortTests.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "executor.h" +#include "stub.h" +#include "taos.h" +#include "tdef.h" +#include "tep.h" +#include "trpc.h" +#include "tvariant.h" + +namespace { +typedef struct { + int32_t startVal; + int32_t count; + int32_t pageRows; +} _info; + +SSDataBlock* getSingleColDummyBlock(void* param) { + _info* pInfo = (_info*) param; + if (--pInfo->count < 0) { + return NULL; + } + + SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(pInfo->pageRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (pInfo->pageRows + 7) / 8)); + + taosArrayPush(pBlock->pDataBlock, &colInfo); + + for (int32_t i = 0; i < pInfo->pageRows; ++i) { + SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); + + int32_t v = ++pInfo->startVal; + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + } + + pBlock->info.rows = pInfo->pageRows; + pBlock->info.numOfCols = 1; + return pBlock; +} + +int32_t docomp(const void* p1, const void* p2, void* param) { + int32_t pLeftIdx = *(int32_t *)p1; + int32_t pRightIdx = *(int32_t *)p2; + + SMsortComparParam *pParam = (SMsortComparParam *)param; + SGenericSource** px = reinterpret_cast(pParam->pSources); + + SArray *pInfo = pParam->orderInfo; + + SGenericSource* pLeftSource = px[pLeftIdx]; + SGenericSource* pRightSource = px[pRightIdx]; + + // this input is exhausted, set the special value to denote this + if (pLeftSource->src.rowIndex == -1) { + return 1; + } + + if (pRightSource->src.rowIndex == -1) { + return -1; + } + + SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; + SSDataBlock* pRightBlock = pRightSource->src.pBlock; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = (SBlockOrderInfo*)TARRAY_GET_ELEM(pInfo, i); + + SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + } + + SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pParam->nullFirst? 1:-1; + } + + if (leftNull) { + return pParam->nullFirst? -1:1; + } + + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + + switch(pLeftColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftv = *(int32_t*)left1; + int32_t rightv = *(int32_t*)right1; + + if (leftv == rightv) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return leftv < rightv? -1 : 1; + } else { + return leftv < rightv? 1 : -1; + } + } + } + default: + assert(0); + } + } + + return 0; +} +} // namespace + +#if 0 +TEST(testCase, inMem_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + int32_t numOfRows = 1000; + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortAddSource(phandle, &numOfRows); + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +TEST(testCase, external_mem_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + + _info* pInfo = (_info*) calloc(1, sizeof(_info)); + pInfo->startVal = 100000; + pInfo->pageRows = 1000; + pInfo->count = 50; + + SGenericSource* ps = static_cast(calloc(1, sizeof(SGenericSource))); + ps->param = pInfo; + + tsortAddSource(phandle, ps); + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +TEST(testCase, ordered_merge_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + int32_t numOfRows = 1000; + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4}; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_MULTISOURCE_MERGE, 1024, 5, &s, 1,"test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortSetComparFp(phandle, docomp); + + for(int32_t i = 0; i < 10; ++i) { + SGenericSource* p = static_cast(calloc(1, sizeof(SGenericSource))); + _info* c = static_cast<_info*>(calloc(1, sizeof(_info))); + c->count = 1; + c->pageRows = 1000; + c->startVal = 0; + + p->param = c; + tsortAddSource(phandle, p); + } + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +#endif + +#pragma GCC diagnostic pop diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 2ff2edf84ebf6754b781c91f3f08940346a9982d..c36fc9c659b11c893b577f17432bdfc881108db6 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -29,9 +29,10 @@ #include "tcompression.h" //#include "queryLog.h" #include "tudf.h" +#include "tep.h" #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) +#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) @@ -3818,7 +3819,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { skey = ekey; } } - assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); +// assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); } else if (type == TSDB_FILL_NEXT) { TSKEY ekey = skey; char* val = NULL; @@ -4395,7 +4396,7 @@ SFunctionFpSet fpSet[1] = { .addInput = count_function, .finalize = doFinalizer, .combine = count_func_merge, - } + }, }; SAggFunctionInfo aggFunc[35] = {{ diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 7d9cb97400732c57bbb52c1abe5a85b47d0d057d..bede8b80fdeb738792eeb3f1685d774287099e3d 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -16,7 +16,7 @@ #include "function.h" #include "os.h" -#include "exception.h" +#include "texception.h" #include "taosdef.h" #include "tmsg.h" #include "tarray.h" diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index db1bb3e7b8dd68f71c6e25bd7de52693d1b3f2d0..653e30cec883b1bb82a635e945cf1ec3077fcfba 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -23,7 +23,7 @@ #include "tfill.h" #include "thash.h" #include "function.h" -#include "common.h" +#include "tcommon.h" #include "ttime.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 40731adc58326da548bd53d4de6f2825efeff021..1aa0c04ec4f28583dcef72e087c888c7599518af 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -12,8 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "os.h" + +#include "tglobal.h" +#include "tcompare.h" #include "taosdef.h" #include "tcompare.h" @@ -221,7 +222,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, } pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; - pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page + pBucket->bufPageSize = 16384 * 4; // 16k per page pBucket->type = dataType; pBucket->bytes = nElemSize; @@ -254,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp"); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; @@ -269,7 +270,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { return; } - destroyResultBuf(pBucket->pBuffer); + destroyDiskbasedBuf(pBucket->pBuffer); tfree(pBucket->pSlots); tfree(pBucket); } @@ -347,7 +348,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data = NULL; } - pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId); + pSlot->info.data = getNewBufPage(pBucket->pBuffer, groupId, &pageId); pSlot->info.pageId = pageId; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 90ad1e15f4b2669620253e52269003abb0e4d1c7..59ed5b79f517f21f365831c6f175ee23842fafdc 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -16,12 +16,15 @@ #ifndef _TD_INDEX_INT_H_ #define _TD_INDEX_INT_H_ +#include "os.h" + #include "index.h" -#include "index_fst.h" #include "taos.h" +#include "tarray.h" #include "tchecksum.h" #include "thash.h" #include "tlog.h" +#include "tutil.h" #ifdef USE_LUCENE #include diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 9db913debfc06f3406aa4d9105709419e5314f1e..1445a1bc562cde1cf51fa3e6cb5df057eaa3cc88 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -15,10 +15,9 @@ #ifndef __INDEX_CACHE_H__ #define __INDEX_CACHE_H__ -#include "index.h" #include "indexInt.h" -#include "tlockfree.h" #include "tskiplist.h" + // ----------------- key structure in skiplist --------------------- /* A data row, the format is like below diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 072ec93e4e648ae55c945f291e0c0d73dc51cbc0..5da0dc537bfbefbe79a430d0220cef3149eb7f7b 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -20,11 +20,12 @@ extern "C" { #endif +#include "indexInt.h" +#include "index_fst_node.h" #include "index_fst_automation.h" #include "index_fst_counting_writer.h" #include "index_fst_registry.h" #include "index_fst_util.h" -#include "tarray.h" #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index be056f38fa183e1460dac83c9fd90b70e69c40e9..786d3eb7d29b404dcbb1a20e4871c8661983e688 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -19,6 +19,7 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_util.h" typedef struct AutomationCtx AutomationCtx; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 86e829aa95f7505537363cec65704e3d8d89e0b2..f8a62467239f55e8461fef562d25ee3cacaf20c1 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -16,12 +16,12 @@ #ifndef __INDEX_FST_COUNTING_WRITER_H__ #define __INDEX_FST_COUNTING_WRITER_H__ +#include "indexInt.h" + #ifdef __cplusplus extern "C" { #endif -#include "tfile.h" - //#define USE_MMAP 1 #define DefaultMem 1024 * 1024 diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 32de7ea6211d4ab449fc6ef0d0fe8355ff380299..4dbf4cea04dcccc44b96d99709f3da287b05e1dd 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_counting_writer.h" #include "index_fst_util.h" diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index f94a21d8dd7fd6887f531793af74f2d5023f66b7..20b70e2866615a07c457fbfaa1919bbe1f17ef6c 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -19,9 +19,9 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_node.h" #include "index_fst_util.h" -#include "tarray.h" typedef struct FstRegistryCell { CompiledAddr addr; diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index 71a378584d401e966d7b2966ff215916c40c8232..f173dd9a15553b9a1997ff5915fc1e50ea9658a7 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -20,8 +20,8 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_common.h" -#include "tarray.h" typedef uint64_t FstType; typedef uint64_t CompiledAddr; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 4618a39197d1d87b1e9d50f895578e9d2dd8fb85..f676651e52262d80cd0c7c2f42ec0fa6c258e43b 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -15,7 +15,6 @@ #ifndef __INDEX_TFILE_H__ #define __INDEX_TFILE_H__ -#include "index.h" #include "indexInt.h" #include "index_fst.h" #include "index_fst_counting_writer.h" diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index 36830a68bcf8156eb2955ce999a444182de82908..985dd657d088c08b36ffac1f2f169420fe8e76b7 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,7 +15,7 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ -#include "tarray.h" +#include "indexInt.h" #ifdef __cplusplus extern "C" { diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 78c7a8d1934ad30c88ca8ab2b7fffd97c266eb88..16d98937780d39e13e683b00818acd7f660d0c3f 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#include "os.h" #include "index_fst_counting_writer.h" #include "indexInt.h" #include "index_fst_util.h" diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 07afc52ecf12bf515e32afca7fc50a44e0a422b3..b28b518fc156b5d7585a2e071b603a186bcd54e7 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "index_fst_registry.h" uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 65118a2bce043dc0e8725f0569a51e19c9bdebdc..698945ae14578d72377c3d30d942da3e7c8c5ec2 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -49,7 +49,6 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size, const std::string& fileName = "/tmp/tindex.tindex") { - tfInit(); _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; @@ -116,7 +115,6 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); } private: @@ -170,7 +168,6 @@ void Performance_fstReadRecords(FstReadMemory* m) { } void checkMillonWriteAndReadOfFst() { - tfInit(); FstWriter* fw = new FstWriter; Performance_fstWriteRecords(fw); delete fw; @@ -181,11 +178,9 @@ void checkMillonWriteAndReadOfFst() { } Performance_fstReadRecords(fr); - tfCleanup(); delete fr; } void checkFstLongTerm() { - tfInit(); FstWriter* fw = new FstWriter; // Performance_fstWriteRecords(fw); @@ -235,12 +230,10 @@ void checkFstLongTerm() { // for (int i = 0; i < result.size(); i++) { // assert(result[i] == i); // check result //} - tfCleanup(); // free(ctx); // delete m; } void checkFstCheckIterator() { - tfInit(); FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); int count = 2; @@ -275,7 +268,6 @@ void checkFstCheckIterator() { free(ctx); delete m; - tfCleanup(); } void fst_get(Fst* fst) { @@ -294,8 +286,6 @@ void fst_get(Fst* fst) { #define NUM_OF_THREAD 10 void validateTFile(char* arg) { - tfInit(); - std::thread threads[NUM_OF_THREAD]; // std::vector threads; TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); @@ -309,12 +299,9 @@ void validateTFile(char* arg) { // wait join threads[i].join(); } - tfCleanup(); } void iterTFileReader(char* path, char* ver) { - tfInit(); - int version = atoi(ver); TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); Iterate* iter = tfileIteratorCreate(reader); @@ -331,7 +318,6 @@ void iterTFileReader(char* path, char* ver) { printf("total size: %d\n term count: %d\n", count, termCount); tfileIteratorDestroy(iter); - tfCleanup(); } int main(int argc, char* argv[]) { diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 9665198b3b651698775c61753dc8473105649cd1..2d3d2f8894415236bb6e79849137fdc931b9bc94 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -24,8 +24,6 @@ static char tindex[PATH_MAX] = {0}; static char tindexDir[PATH_MAX] = {0}; static void EnvInit() { - tfInit(); - std::string path = dir; taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); @@ -136,7 +134,6 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - // tfCleanup(); } private: diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index bbcc654ae26fe8b4ffbbf01072c893bbea067876..a50e91b0944870d4c658c5d51ec38028aa0d98a6 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -420,7 +420,6 @@ class IndexTFileEnv : public ::testing::Test { virtual void SetUp() { taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); - tfInit(); fObj = new TFileObj(dir, colName); } @@ -428,7 +427,6 @@ class IndexTFileEnv : public ::testing::Test { // indexClose(index); // indexeptsDestroy(opts); delete fObj; - tfCleanup(); // tfileWriterDestroy(twrite); } TFileObj* fObj; @@ -800,13 +798,10 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: virtual void SetUp() { - tfInit(); index = new IndexObj(); - // } virtual void TearDown() { delete index; - tfCleanup(); } IndexObj* index; }; diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e61375b2023670db09338a8bd7a844e22c5f5788..5c81cce94aa66f8b082e686c509fef483eff13e2 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -105,6 +105,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_RAW_EXPR: res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; + case QUERY_NODE_TARGET: + res = walkNode(((STargetNode*)pNode)->pExpr, order, walker, pContext); + break; default: break; } @@ -228,6 +231,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_RAW_EXPR: res = rewriteNode(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; + case QUERY_NODE_TARGET: + res = rewriteNode(&(((STargetNode*)pNode)->pExpr), order, rewriter, pContext); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4b0f4394a0650423e50533b18e2a84148d53c69e..c34b0e9d72327c72deee8a34a8281a35fa987302 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -103,9 +103,16 @@ SNode* nodesMakeNode(ENodeType type) { static EDealRes destroyNode(SNode** pNode, void* pContext) { switch (nodeType(*pNode)) { - case QUERY_NODE_VALUE: - tfree(((SValueNode*)(*pNode))->literal); + case QUERY_NODE_VALUE: { + SValueNode* pValue = (SValueNode*)*pNode; + + tfree(pValue->literal); + if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + tfree(pValue->datum.p); + } + break; + } case QUERY_NODE_LOGIC_CONDITION: nodesDestroyList(((SLogicConditionNode*)(*pNode))->pParameterList); break; @@ -115,6 +122,9 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { case QUERY_NODE_GROUPING_SET: nodesDestroyList(((SGroupingSetNode*)(*pNode))->pParameterList); break; + case QUERY_NODE_NODE_LIST: + nodesDestroyList(((SNodeListNode*)(*pNode))->pNodeList); + break; default: break; } @@ -123,6 +133,9 @@ static EDealRes destroyNode(SNode** pNode, void* pContext) { } void nodesDestroyNode(SNode* pNode) { + if (NULL == pNode) { + return; + } nodesRewriteNodePostOrder(&pNode, destroyNode, NULL); } @@ -203,6 +216,7 @@ void nodesDestroyList(SNodeList* pList) { if (NULL == pList) { return; } + SListCell* pNext = pList->pHead; while (NULL != pNext) { pNext = nodesListErase(pList, pNext); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 242f659288880de268d3f4edb8b10375ff50527e..fa713d5fb42cd8ddda7ccb3b9037f4ca31f88891 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -449,10 +449,10 @@ tagitemlist1(A) ::= tagitemlist1(X) COMMA tagitem1(Y). { taosArrayPush(X, &Y); A tagitemlist1(A) ::= tagitem1(Y). { A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &Y); } %type tagitem1 {SToken} -tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; } +tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} tagitem1(A) ::= INTEGER(X). { A = X; } tagitem1(A) ::= FLOAT(X). { A = X; } tagitem1(A) ::= STRING(X). { A = X; } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 93848e2c3e042fde74c52103a919f1e88f676781..e7afd2e49c6b880d1847c77068e64df71fbf02e3 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -16,7 +16,7 @@ #if 0 #include "plannerInt.h" -#include "exception.h" +#include "texception.h" #include "parser.h" #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 753ac75860539574f6691dea3e500c241f7dc723..8bc29656d41300a9db7c7ba6d22a22c83c72001d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -198,7 +198,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl } int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { - int32_t code = -1; + int32_t code = 0; STableMetaRsp metaRsp = {0}; if (NULL == output || NULL == msg || msgSize <= 0) { @@ -216,7 +216,7 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_META_OVER; } - if (!tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { + if (0 != strcmp(metaRsp.dbFName, TSDB_INFORMATION_SCHEMA_DB) && !tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { code = TSDB_CODE_TSC_INVALID_VALUE; goto PROCESS_META_OVER; } @@ -254,11 +254,11 @@ PROCESS_META_OVER: void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; - queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)] = queryProcessTableMetaRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cebe8178d9303e330bbbd21a211fe34c95ed3b2a..7a6fa4cfd78f1c9278931cfc300a85fc7f4a05f8 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,5 +1,5 @@ #include "qworker.h" -#include +#include "tcommon.h" #include "executor.h" #include "planner.h" #include "query.h" diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index a1b431fd7d92b3b6f1dcb41f6ecb136315527c53..bdb9617d0dc9ae8f44ad2d3d34aa8022307aeea4 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -1,5 +1,5 @@ #include "qworker.h" -#include +#include "tcommon.h" #include "executor.h" #include "planner.h" #include "query.h" diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index a0d4bc680cc1742ee76dffa0b139f83bb079ff3e..b4fe9a67caa2e8bb354d486cd3ff51427266b290 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -22,7 +22,7 @@ extern "C" { #include "thash.h" #include "tname.h" -#include "common.h" +#include "tcommon.h" #include "scalar.h" #include "querynodes.h" #include "query.h" @@ -253,6 +253,7 @@ typedef struct SFilterInfo { uint32_t *blkUnits; int8_t *blkUnitRes; void *pTable; + SArray *blkList; SFilterPCtx pctx; } SFilterInfo; @@ -306,7 +307,7 @@ typedef struct SFilterInfo { #define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) #define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) #define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGet(((SColumnInfoData *)(fi)->data), (ri))) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 41ee90667e012dde68d9c7179a05b3595001182f..b79221cf82c50bdd560ef40bb8dab90b48c206b4 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -18,13 +18,13 @@ #ifdef __cplusplus extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "thash.h" #include "query.h" typedef struct SScalarCtx { int32_t code; - SSDataBlock *pSrc; + SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ } SScalarCtx; @@ -44,10 +44,13 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) - +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx); +bool sclIsNull(SScalarParam* param, int32_t idx); +void sclSetNull(SScalarParam* param, int32_t idx); +void sclFreeParam(SScalarParam *param); #ifdef __cplusplus } #endif -#endif // TDENGINE_SCALARINT_H \ No newline at end of file +#endif // TDENGINE_SCALARINT_H diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 55c482874511c86ef630c189df751f6b018b71d7..09b813359a6ef98f311577f04b0798dac1d272b4 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -22,7 +22,11 @@ extern "C" { #include "sclfunc.h" -typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); +typedef double (*_mathFunc)(double, double, bool *); + + +typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4d6656b061d1221452332d43b72c57c190bd8615..e5be22c0a8862150e15ac0aa6628341e6f5c30c4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -18,7 +18,9 @@ //#include "queryLog.h" #include "tcompare.h" #include "filterInt.h" +#include "sclInt.h" #include "filter.h" +#include "tep.h" OptrStr gOptrStr[] = { {0, "invalid"}, @@ -1677,7 +1679,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); - + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); @@ -2776,14 +2778,14 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = colDataGet((SColumnInfoData *)cunit->colData, i); + void *colData = colDataGetData((SColumnInfoData *)cunit->colData, i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { + if (colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -2874,18 +2876,18 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData){ // for json->'key' is null (*p)[i] = 1; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ (*p)[i] = 0; } }else{ - (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { all = false; @@ -2908,19 +2910,19 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData) { // for json->'key' is not null (*p)[i] = 0; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ // for json->'key' is not null (*p)[i] = 1; } }else { - (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { @@ -2949,9 +2951,9 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD } for (int32_t i = 0; i < numOfRows; ++i) { - void *colData = colDataGet((SColumnInfoData *)info->cunits[0].colData, i); + void *colData = colDataGetData((SColumnInfoData *)info->cunits[0].colData, i); - if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[0].colData, 0, i, NULL)) { all = false; continue; } @@ -2980,8 +2982,8 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = colDataGet((SColumnInfoData *)info->cunits[uidx].colData, i); - if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)) { (*p)[i] = 0; all = false; continue; @@ -3031,14 +3033,14 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = colDataGet((SColumnInfoData *)(cunit->colData), i); + void *colData = colDataGetData((SColumnInfoData *)(cunit->colData), i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); //} else { uint8_t optr = cunit->optr; - if (colData == NULL || isNull(colData, cunit->dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -3668,9 +3670,17 @@ _return: bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + SArray *pList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(pList, &pSrc); + + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); + + taosArrayDestroy(pList); + + *p = output.orig.data; + output.orig.data = NULL; - *p = output.data; + sclFreeParam(&output); int8_t *r = output.data; for (int32_t i = 0; i < output.num; ++i) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 742c7fd706cc4cfb397a7b4150ad5f1a2a8e552f..0245226dfcdfdbd360ab1a6e18d20d95839abe1d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,10 +1,11 @@ #include "nodes.h" -#include "common.h" +#include "tcommon.h" #include "querynodes.h" #include "function.h" #include "functionMgt.h" #include "sclvector.h" #include "sclInt.h" +#include "tep.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -87,6 +88,26 @@ _return: SCL_RET(code); } +FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { + if (param->dataInBlock) { + return colDataIsNull(param->orig.columnData, 0, idx, NULL); + } + + return param->bitmap ? colDataIsNull_f(param->bitmap, idx) : false; +} + +FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { + if (NULL == param->bitmap) { + param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + if (NULL == param->bitmap) { + sclError("calloc %d failed", param->num); + return; + } + } + + colDataSetNull_f(param->bitmap, idx); +} + void sclFreeRes(SHashObj *res) { SScalarParam *p = NULL; @@ -95,7 +116,7 @@ void sclFreeRes(SHashObj *res) { p = (SScalarParam *)pIter; if (p) { - tfree(p->data); + sclFreeParam(p); } pIter = taosHashIterate(res, pIter); @@ -104,19 +125,54 @@ void sclFreeRes(SHashObj *res) { taosHashCleanup(res); } +void sclFreeParamNoData(SScalarParam *param) { + tfree(param->bitmap); +} + + void sclFreeParam(SScalarParam *param) { - tfree(param->data); + sclFreeParamNoData(param); + + if (!param->dataInBlock) { + if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { + taosHashCleanup((SHashObj *)param->orig.data); + } else { + tfree(param->orig.data); + } + } +} + + +int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { + if (TSDB_DATA_TYPE_NULL == pNode->node.resType.type) { + return TSDB_CODE_SUCCESS; + } + + *res = malloc(pNode->node.resType.bytes); + if (NULL == (*res)) { + sclError("malloc %d failed", pNode->node.resType.bytes); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); + + return TSDB_CODE_SUCCESS; } int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; + //SCL_ERR_RET(sclCopyValueNodeValue(valueNode, ¶m->data)); param->data = nodesGetValueFromNode(valueNode); + param->orig.data = param->data; param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; - param->colData = false; + if (TSDB_DATA_TYPE_NULL == param->type) { + sclSetNull(param, 0); + } + param->dataInBlock = false; break; } @@ -128,34 +184,44 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->orig.data = param->data; param->num = 1; param->type = SCL_DATA_TYPE_DUMMY_HASH; - param->colData = false; - + param->dataInBlock = false; + + if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { + taosHashCleanup(param->orig.data); + param->orig.data = NULL; + sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } break; } case QUERY_NODE_COLUMN: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + if (NULL == ctx->pBlockList) { + sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } SColumnNode *ref = (SColumnNode *)node; - if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { - sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); + if (ref->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", ref->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); - if (IS_VAR_DATA_TYPE(columnData->info.type)) { - param->data = columnData; - param->colData = true; - } else { - param->data = columnData->pData; - param->colData = false; + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->dataBlockId); + + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId); + param->data = NULL; + param->orig.columnData = columnData; + param->dataInBlock = true; - param->num = ctx->pSrc->info.rows; + param->num = block->info.rows; param->type = columnData->info.type; param->bytes = columnData->info.bytes; @@ -163,11 +229,6 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_OPERATOR: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -192,20 +253,26 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) { SScalarParam *param = NULL; - for (int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < listNum; ++i) { param = params + i; if (1 == param->num) { continue; } - if (IS_VAR_DATA_TYPE(param->type)) { - param->data = (char *)(param->data) + varDataTLen(param->data); + if (param->dataInBlock) { + param->data = colDataGetData(param->orig.columnData, idx); + } else if (idx) { + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } } else { - param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + param->data = param->orig.data; } } @@ -281,8 +348,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( -"fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -296,24 +362,27 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( -"scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } @@ -348,12 +417,14 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; - void *data = output->data; - bool value = false; for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + for (int32_t m = 0; m < node->pParameterList->length; ++m) { GET_TYPED_DATA(value, bool, params[m].type, params[m].data); @@ -367,17 +438,14 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } *(bool *)output->data = value; - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } - output->data = data; - - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); SCL_RET(code); } @@ -397,6 +465,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); @@ -404,23 +473,27 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); - - return TSDB_CODE_SUCCESS; + OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); _return: + + for (int32_t i = 0; i < paramNum; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } -EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { +EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -428,7 +501,7 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -449,12 +522,12 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { +EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecLogic(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -462,7 +535,7 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -483,12 +556,12 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { +EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecOperator(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -496,7 +569,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -523,28 +596,29 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - return sclRewriteFunction(pNode, pContext); + return sclRewriteFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { - return sclRewriteLogic(pNode, pContext); + return sclRewriteLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, pContext); + return sclRewriteOperator(pNode, ctx); } sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); - - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -EDealRes sclWalkFunction(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)pNode; SScalarParam output = {0}; @@ -562,8 +636,7 @@ EDealRes sclWalkFunction(SNode* pNode, void* pContext) { } -EDealRes sclWalkLogic(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)pNode; SScalarParam output = {0}; @@ -581,8 +654,7 @@ EDealRes sclWalkLogic(SNode* pNode, void* pContext) { } -EDealRes sclWalkOperator(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)pNode; SScalarParam output = {0}; @@ -599,27 +671,69 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { + STargetNode *target = (STargetNode *)pNode; + + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); + ctx->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + for (int32_t i = 0; i < res->num; ++i) { + sclMoveParamListData(res, 1, i); + + colDataAppend(col, i, res->data, sclIsNull(res, i)); + } + + sclFreeParam(res); + + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + + return DEAL_RES_CONTINUE; +} + EDealRes sclCalcWalker(SNode* pNode, void* pContext) { if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } + + SScalarCtx *ctx = (SScalarCtx *)pContext; if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - return sclWalkFunction(pNode, pContext); + return sclWalkFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { - return sclWalkLogic(pNode, pContext); + return sclWalkLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, pContext); + return sclWalkOperator(pNode, ctx); } - sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); + if (QUERY_NODE_TARGET == nodeType(pNode)) { + return sclWalkTarget(pNode, ctx); + } - SScalarCtx *ctx = (SScalarCtx *)pContext; + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; @@ -634,26 +748,33 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { } int32_t code = 0; + SScalarCtx ctx = {0}; + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); - if (code) { - nodesDestroyNode(pNode); - SCL_ERR_RET(code); - } + SCL_ERR_JRET(ctx.code); *pRes = pNode; - SCL_RET(code); +_return: + + sclFreeRes(ctx.pRes); + + return code; } -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { - if (NULL == pNode || NULL == pSrc || NULL == pDst) { +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { + if (NULL == pNode || NULL == pBlockList) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; + SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList}; ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -663,23 +784,28 @@ int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); - if (ctx.code) { - nodesDestroyNode(pNode); - sclFreeRes(ctx.pRes); - SCL_ERR_RET(ctx.code); - } + SCL_ERR_JRET(ctx.code); - SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); - if (NULL == res) { - sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + if (pDst) { + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + sclMoveParamListData(res, 1, 0); + + *pDst = *res; + + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } - - *pDst = *res; - nodesDestroyNode(pNode); +_return: + + //nodesDestroyNode(pNode); + sclFreeRes(ctx.pRes); - return TSDB_CODE_SUCCESS; + return code; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 85af663313eeb64669d4066274202308a320297c..3c431ff33fde08326adf1797a183452197bc0505 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -226,7 +226,7 @@ void* getVectorValueAddr_default(void *src, int32_t index) { return src; } void* getVectorValueAddr_VAR(void *src, int32_t index) { - return colDataGet((SColumnInfoData *)src, index); + return colDataGetData((SColumnInfoData *)src, index); } _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { @@ -261,15 +261,89 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } +static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t outType) { + int64_t value = strtoll(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t outType) { + uint64_t value = strtoull(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t outType) { + double value = strtod(buf, NULL); + SET_TYPED_DATA(pOut->data, outType, value); +} + + +int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { + int32_t bufSize = 0; + char *tmp = NULL; + _bufConverteFunc func = NULL; + + if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType || TSDB_DATA_TYPE_BOOL == outType) { + func = varToSigned; + } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + func = varToUnsigned; + } else if (IS_FLOAT_TYPE(outType)) { + func = varToFloat; + } else { + sclError("invalid convert outType:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; + } + + if (TSDB_DATA_TYPE_BINARY == inType) { + if (varDataLen(pIn->data) >= bufSize) { + bufSize = varDataLen(pIn->data) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); + tmp[varDataLen(pIn->data)] = 0; + } else { + if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + } + + (*func)(tmp, pOut, outType); + } + + tfree(tmp); + + return TSDB_CODE_SUCCESS; +} int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int16_t inType = pIn->type; int16_t inBytes = pIn->bytes; - char *input = pIn->data; int16_t outType = pOut->type; int16_t outBytes = pOut->bytes; - char *output = pOut->data; + if (inType == TSDB_DATA_TYPE_BINARY || inType == TSDB_DATA_TYPE_NCHAR) { + return vectorConvertFromVarData(pIn, pOut, inType, outType); + } + switch (outType) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -277,239 +351,55 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } + + int64_t value = 0; - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + GET_TYPED_DATA(value, int64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + uint64_t value = 0; + + GET_TYPED_DATA(value, uint64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + double value = 0; + + GET_TYPED_DATA(value, double, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; default: @@ -595,6 +485,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut1->orig.data = paramOut1->data; code = vectorConvertImpl(param1, paramOut1); if (code) { @@ -612,6 +503,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p tfree(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { @@ -624,24 +516,21 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } -void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, _mathFunc func) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; - SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + double leftv = 0, rightv = 0; + bool isNull = false; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -651,64 +540,237 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; + if (vectorConvertImpl(pRight, &rightParam)) { + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); + return; } + pRight = &rightParam; + } + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); +} + +double mathAdd(double leftv, double rightv, bool *isNull) { + return leftv + rightv; +} + +double mathSub(double leftv, double rightv, bool *isNull) { + return leftv - rightv; +} + +double mathMultiply(double leftv, double rightv, bool *isNull) { + return leftv * rightv; +} + +double mathDivide(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv / rightv; +} + +double mathRemainder(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv - ((int64_t)(leftv / rightv)) * rightv; +} + + +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathAdd); +} + +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathSub); +} + +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathMultiply); +} + +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathDivide); +} + +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathRemainder); +} + +#if 0 +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + leftParam.orig.data = leftParam.data; + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); + return; + } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -718,12 +780,8 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -733,62 +791,75 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); - } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -798,12 +869,8 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -813,64 +880,76 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); } - } + } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -880,12 +959,8 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -895,71 +970,73 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } + } - SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) / getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) / getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -969,12 +1046,8 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -984,94 +1057,98 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double * output = (double *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pLeft->num == 1) { double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pRight->num == 1) { double right = getVectorDoubleValueFnRight(pRight->data, 0); - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double left = getVectorDoubleValueFnLeft(pLeft->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } +#endif + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; @@ -1119,28 +1196,24 @@ void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); } } - } -void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1149,80 +1222,95 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1231,131 +1319,139 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); bool res = false; - - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { - getVectorValueAddrFnRight = getVectorValueAddr_default; - } else { - getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - } - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pLeft->num == 1) { - void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); + sclMoveParamListData(pLeft, 1, 0); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { - void *rightData = getVectorValueAddrFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + sclMoveParamListData(pRight, 1, 0); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } } -void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { SScalarParam pLeftOut = {0}; SScalarParam pRightOut = {0}; @@ -1377,118 +1473,103 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t param2 = pRight; } - vectorCompareImpl(param1, param2, out, _ord, optr); + vectorCompareImpl(param1, param2, pOut, _ord, optr); + + sclFreeParam(&pLeftOut); + sclFreeParam(&pRightOut); } -void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } -void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_EQUAL); } -void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_THAN); } -void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_EQUAL); } -void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_EQUAL); } -void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_EQUAL); } -void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_IN); } -void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_IN); } -void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LIKE); } -void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_LIKE); } -void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_MATCH); } -void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } -void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } -void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - - bool *output = (bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } -} -void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; +} - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - - vectorConvertImpl(pLeft, &output); +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertImpl(pLeft, pOut); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index a0618cc6624676b89f2586747952a3451a09b5b3..fafc1ea42e860a8c410d75c1e58738f3c1e9d79e 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -51,6 +51,7 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); @@ -75,16 +76,20 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + static uint64_t dbidx = 0; + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; + + sprintf(rnode->dbName, "%" PRIu64, dbidx++); if (NULL == block) { rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *pNode = (SNode *)rnode; return; @@ -99,7 +104,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -109,18 +114,25 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *block = res; } else { @@ -130,14 +142,26 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; - idata.pData = (char *)value; + idata.info.colId = 1 + idx; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; - rnode->colId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -196,11 +220,11 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { } TEST(timerangeTest, greater) { - SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); @@ -212,6 +236,8 @@ TEST(timerangeTest, greater) { ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, INT64_MAX); + filterFreeInfo(filter); + nodesDestroyNode(opNode1); } TEST(timerangeTest, greater_and_lower) { @@ -219,10 +245,10 @@ TEST(timerangeTest, greater_and_lower) { bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); SNode *list[2] = {0}; @@ -239,6 +265,8 @@ TEST(timerangeTest, greater_and_lower) { ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, tbig); + filterFreeInfo(filter); + nodesDestroyNode(logicNode); } @@ -250,7 +278,7 @@ TEST(columnTest, smallint_column_greater_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -291,6 +319,10 @@ TEST(columnTest, smallint_column_greater_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + blockDataDestroy(src); + nodesDestroyNode(opNode); } TEST(columnTest, int_column_greater_smallint_value) { @@ -301,7 +333,7 @@ TEST(columnTest, int_column_greater_smallint_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -342,6 +374,10 @@ TEST(columnTest, int_column_greater_smallint_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -353,7 +389,7 @@ TEST(columnTest, int_column_in_double_list) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -384,7 +420,10 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } - + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -416,7 +455,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -446,6 +485,10 @@ TEST(columnTest, binary_column_in_binary_list) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -465,7 +508,7 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); @@ -491,6 +534,10 @@ TEST(columnTest, binary_column_like_binary) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -499,20 +546,21 @@ TEST(columnTest, binary_column_is_null) { char leftv[5][5]= {0}; SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); SFilterInfo *filter = NULL; @@ -534,6 +582,10 @@ TEST(columnTest, binary_column_is_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(columnTest, binary_column_is_not_null) { @@ -543,17 +595,18 @@ TEST(columnTest, binary_column_is_not_null) { SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -576,6 +629,10 @@ TEST(columnTest, binary_column_is_not_null) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -588,8 +645,8 @@ TEST(opTest, smallint_column_greater_int_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); SFilterInfo *filter = NULL; @@ -611,6 +668,10 @@ TEST(opTest, smallint_column_greater_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -623,7 +684,7 @@ TEST(opTest, smallint_value_add_int_column) { SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -646,6 +707,10 @@ TEST(opTest, smallint_value_add_int_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -663,8 +728,8 @@ TEST(opTest, bigint_column_multi_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -687,6 +752,10 @@ TEST(opTest, bigint_column_multi_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_and_binary_column) { @@ -702,8 +771,8 @@ TEST(opTest, smallint_column_and_binary_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -726,6 +795,10 @@ TEST(opTest, smallint_column_and_binary_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_or_float_column) { @@ -736,8 +809,8 @@ TEST(opTest, smallint_column_or_float_column) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -760,6 +833,10 @@ TEST(opTest, smallint_column_or_float_column) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -772,7 +849,7 @@ TEST(opTest, smallint_column_or_double_value) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); @@ -796,6 +873,10 @@ TEST(opTest, smallint_column_or_double_value) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -814,7 +895,7 @@ TEST(opTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); @@ -837,10 +918,16 @@ TEST(opTest, binary_column_is_true) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(filterModelogicTest, diff_columns_and_or_and) { + flttInitLogFile(); + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; SNode *logicNode1 = NULL, *logicNode2 = NULL; double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; @@ -851,12 +938,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -866,12 +953,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -903,6 +990,10 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } TEST(filterModelogicTest, same_column_and_or_and) { @@ -916,11 +1007,12 @@ TEST(filterModelogicTest, same_column_and_or_and) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -929,11 +1021,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { list = nodesMakeList(); - + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -965,6 +1059,10 @@ TEST(filterModelogicTest, same_column_and_or_and) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } @@ -979,12 +1077,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -994,12 +1092,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -1031,6 +1129,10 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } TEST(filterModelogicTest, same_column_or_and_or) { @@ -1044,11 +1146,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -1058,10 +1161,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { list = nodesMakeList(); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); @@ -1093,6 +1198,10 @@ TEST(filterModelogicTest, same_column_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } @@ -1110,13 +1219,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { SNodeList* list = nodesMakeList(); int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); - flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); - flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -1125,13 +1234,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { list = nodesMakeList(); - flttMakeColRefNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); - flttMakeColRefNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); - flttMakeColRefNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); - flttMakeColRefNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); @@ -1162,6 +1271,10 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 086594ddf542c0d1442992872a9c9f76dac16c38..b9aef99088bef2fefb98a7447cd77c6945c1ba76 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -44,18 +44,53 @@ namespace { int64_t scltLeftV = 21, scltRightV = 10; double scltLeftVd = 21.0, scltRightVd = 10.0; +void scltFreeDataBlock(void *block) { + blockDataDestroy(*(SSDataBlock **)block); +} + void scltInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } +void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { + if (newBlock) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 1; + res->info.rows = rows; + res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + taosArrayPush(res->pDataBlock, &idata); + taosArrayPush(pBlockList, &res); + + blockDataEnsureCapacity(res, rows); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + *slotId = 0; + } else { + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); + res->info.numOfCols++; + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + blockDataEnsureColumnCapacity(&idata, rows); + + taosArrayPush(res->pDataBlock, &idata); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + *slotId = taosArrayGetSize(res->pDataBlock) - 1; + } +} void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { SNode *node = nodesMakeNode(QUERY_NODE_VALUE); @@ -65,7 +100,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { if (IS_VAR_DATA_TYPE(dataType)) { vnode->datum.p = (char *)malloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); - vnode->node.resType.bytes = varDataLen(value); + vnode->node.resType.bytes = varDataTLen(value); } else { vnode->node.resType.bytes = tDataTypes[dataType].bytes; assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); @@ -74,7 +109,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode *rnode = (SColumnNode *)node; rnode->node.resType.type = dataType; @@ -90,7 +125,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -100,18 +135,25 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->colId = 55; + rnode->colId = 3; *block = res; } else { @@ -121,12 +163,26 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; - idata.pData = (char *)value; + idata.info.colId = 1 + idx; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); + res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; - rnode->colId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -171,6 +227,17 @@ void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi *pNode = (SNode *)onode; } +void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode *snode) { + SNode *node = nodesMakeNode(QUERY_NODE_TARGET); + STargetNode *onode = (STargetNode *)node; + onode->pExpr = snode; + onode->dataBlockId = dataBlockId; + onode->slotId = slotId; + + *pNode = (SNode *)onode; +} + + } @@ -187,6 +254,7 @@ TEST(constantTest, bigint_add_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, double_sub_bigint) { @@ -202,6 +270,7 @@ TEST(constantTest, double_sub_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, tinyint_and_smallint) { @@ -217,6 +286,7 @@ TEST(constantTest, tinyint_and_smallint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); + nodesDestroyNode(res); } TEST(constantTest, bigint_or_double) { @@ -232,6 +302,7 @@ TEST(constantTest, bigint_or_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_or_binary) { @@ -250,6 +321,7 @@ TEST(constantTest, int_or_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); + nodesDestroyNode(res); } @@ -266,6 +338,7 @@ TEST(constantTest, int_greater_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_greater_equal_binary) { @@ -284,6 +357,7 @@ TEST(constantTest, int_greater_equal_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, tinyint_lower_ubigint) { @@ -299,11 +373,13 @@ TEST(constantTest, tinyint_lower_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); + nodesDestroyNode(res); } TEST(constantTest, usmallint_lower_equal_ubigint) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int64_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -315,11 +391,13 @@ TEST(constantTest, usmallint_lower_equal_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv <= rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint1) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int16_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -331,6 +409,7 @@ TEST(constantTest, int_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint2) { @@ -347,6 +426,7 @@ TEST(constantTest, int_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint1) { @@ -363,6 +443,7 @@ TEST(constantTest, int_not_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint2) { @@ -379,6 +460,7 @@ TEST(constantTest, int_not_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } @@ -406,6 +488,7 @@ TEST(constantTest, int_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_in_smallint2) { @@ -431,6 +514,7 @@ TEST(constantTest, int_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint1) { @@ -454,6 +538,7 @@ TEST(constantTest, int_not_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint2) { @@ -479,6 +564,7 @@ TEST(constantTest, int_not_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary1) { @@ -499,6 +585,7 @@ TEST(constantTest, binary_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary2) { @@ -519,6 +606,7 @@ TEST(constantTest, binary_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary1) { @@ -539,6 +627,7 @@ TEST(constantTest, binary_not_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary2) { @@ -559,6 +648,7 @@ TEST(constantTest, binary_not_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary1) { @@ -579,6 +669,7 @@ TEST(constantTest, binary_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary2) { @@ -599,6 +690,7 @@ TEST(constantTest, binary_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary1) { @@ -619,6 +711,7 @@ TEST(constantTest, binary_not_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary2) { @@ -639,6 +732,7 @@ TEST(constantTest, binary_not_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null1) { @@ -654,12 +748,13 @@ TEST(constantTest, int_is_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -669,6 +764,7 @@ TEST(constantTest, int_is_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null1) { @@ -684,12 +780,13 @@ TEST(constantTest, int_is_not_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -699,6 +796,7 @@ TEST(constantTest, int_is_not_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true1) { @@ -716,6 +814,7 @@ TEST(constantTest, int_add_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true2) { @@ -733,6 +832,7 @@ TEST(constantTest, int_add_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } @@ -751,6 +851,7 @@ TEST(constantTest, int_greater_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_greater_int_is_true2) { @@ -768,6 +869,7 @@ TEST(constantTest, int_greater_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, greater_and_lower) { @@ -792,30 +894,45 @@ TEST(constantTest, greater_and_lower) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(columnTest, smallint_value_add_int_column) { + scltInitLogFile(); + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int32_t leftv = 1; int16_t rightv[5]= {0, -5, -4, 23, 100}; double eRes[5] = {1.0, -4, -3, 24, 101}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, bigint_column_multi_binary_column) { @@ -829,20 +946,32 @@ TEST(columnTest, bigint_column_multi_binary_column) { } double eRes[5] = {0, 2, 6, 12, 20}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_and_binary_column) { @@ -856,20 +985,31 @@ TEST(columnTest, smallint_column_and_binary_column) { } int64_t eRes[5] = {0, 0, 2, 0, 4}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_float_column) { @@ -878,20 +1018,31 @@ TEST(columnTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; int64_t eRes[5] = {3, 3, 7, 5, 7}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_double_value) { @@ -900,20 +1051,31 @@ TEST(columnTest, smallint_column_or_double_value) { double rightv= 10.2; int64_t eRes[5] = {11, 10, 11, 14, 15}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_greater_double_value) { @@ -922,20 +1084,31 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; bool eRes[5] = {false, false, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, int_column_in_double_list) { @@ -944,9 +1117,8 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -956,22 +1128,33 @@ TEST(columnTest, int_column_in_double_list) { nodesListAppend(list, pRight); scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -994,7 +1177,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -1005,14 +1188,26 @@ TEST(columnTest, binary_column_in_binary_list) { scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_like_binary) { @@ -1020,7 +1215,6 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -1031,28 +1225,41 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } + TEST(columnTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -1063,80 +1270,117 @@ TEST(columnTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, greater_and_lower) { @@ -1148,30 +1392,40 @@ TEST(columnTest, greater_and_lower) { int32_t v4[5]= {5, 3, 4, 2, 6}; bool eRes[5] = {false, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(v1)/sizeof(v1[0]); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v2); scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, v3); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v4); scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); list[0] = opNode1; list[1] = opNode2; scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); - int32_t code = scalarCalculate(logicNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&logicNode, dataBlockId, slotId, logicNode); + + int32_t code = scalarCalculate(logicNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(logicNode); } - int main(int argc, char** argv) { srand(time(NULL)); testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 2831a83905edac61772aa327233e4661823a5a92..03f81218ae4d6a9754926e736d1e2b4429aa0ae2 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -578,7 +578,7 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod atomic_store_32(&pJob->errCode, errCode); } - if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + if (atomic_load_8(&pJob->userFetch) || pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } @@ -638,7 +638,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) { + if (pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 4624e05f101cb3c079a4e632f939d8beb9634a9e..ca0c7561412f9614530e85129f0604f5c3da5db7 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -16,7 +16,7 @@ #ifndef _TD_WAL_INT_H_ #define _TD_WAL_INT_H_ -#include "compare.h" +#include "tcompare.h" #include "taoserror.h" #include "tchecksum.h" #include "tcoding.h" diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d3cd23f284cd8edef6f62546290ae678f5b940dd..b1caeab3b1396ae601326217c956133d776d54f8 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -14,10 +14,9 @@ */ #define _DEFAULT_SOURCE -#include "compare.h" +#include "tcompare.h" #include "os.h" #include "taoserror.h" -#include "tfile.h" #include "tref.h" #include "walInt.h" @@ -40,15 +39,9 @@ int32_t walInit() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); if (old == 1) return 0; - int code = tfInit(); - if (code != 0) { - wError("failed to init tfile since %s", tstrerror(code)); - atomic_store_8(&tsWal.inited, 0); - return code; - } tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - code = walCreateThread(); + int32_t code = walCreateThread(); if (code != 0) { wError("failed to init wal module since %s", tstrerror(code)); atomic_store_8(&tsWal.inited, 0); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 159d2817591900bb02f7071190f5b0651c5d2ebf..9e1ffeae7f83deb4692ba221ad3f334fcc6965a2 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tfile.h" #include "walInt.h" #include "taoserror.h" diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 6b3abcd0f98ca15209f1fbeacab35ff8d83f14e8..140b7ddc32614662ceb86daae8f3beafb1ff8926 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tfile.h" #include "tref.h" #include "walInt.h" diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index ba071340034bbae320e9940efbf5032d387614fd..4b1f0ba30637a01a78a01f9db5a4fd533e1c87dd 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,7 +18,6 @@ #include "os.h" #include "taoserror.h" #include "tchecksum.h" -#include "tfile.h" #include "walInt.h" int32_t walCommit(SWal *pWal, int64_t ver) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 2d8314058ec4122cacfb3eab1109c06ddd3d7657..2d77df9b435b78c7f237cdc2b132efc1c8473039 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -190,39 +190,52 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return NULL; #else - int access = O_BINARY; - char *mode = NULL; - access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; - if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { - access |= O_RDWR; - mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; - } else if (tdFileOptions & TD_FILE_WRITE) { - access |= O_WRONLY; - mode = (tdFileOptions & TD_FILE_TEXT) ? "wt" : "wb"; - } else if (tdFileOptions & TD_FILE_READ) { - access |= O_RDONLY; - mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; - } - access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; - access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; - access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; - access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + int fd = -1; + FILE *fp = NULL; + if (tdFileOptions & TD_FILE_STREAM) { + char *mode = NULL; + if (tdFileOptions & TD_FILE_APPEND) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+"; + }else if (tdFileOptions & TD_FILE_TRUNC) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt+" : "wb+"; + }else if ((tdFileOptions & TD_FILE_READ) && !(tdFileOptions & TD_FILE_WRITE)) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt" : "rb"; + }else { + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + } + assert(!(tdFileOptions & TD_FILE_EXCL)); + fp = fopen(path, mode); + if (fp == NULL) { + return NULL; + } + } else { + int access = O_BINARY; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { + access |= O_RDWR; + } else if (tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + } else if (tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd == -1) { + return NULL; + } + } + if (tdFileOptions & TD_FILE_AUTO_DEL) { autoDelFileListAdd(path); } - int fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); - if (fd == -1) { - return NULL; - } - FILE *fp = fdopen(fd, mode); - if (fp == NULL) { - close(fd); - return NULL; - } + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); if (pFile == NULL) { - close(fd); - fclose(fp); + if (fd >= 0) close(fd); + if (fp != NULL) fclose(fp); return NULL; } pFile->fd = fd; @@ -239,11 +252,16 @@ int64_t taosCloseFile(TdFilePtr *ppFile) { if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { return 0; } - fflush((*ppFile)->fp); - fsync((*ppFile)->fd); - close((*ppFile)->fd); - (*ppFile)->fd = -1; - (*ppFile)->fp = NULL; + if ((*ppFile)->fp != NULL) { + fflush((*ppFile)->fp); + fclose((*ppFile)->fp); + (*ppFile)->fp = NULL; + } + if ((*ppFile)->fd >= 0) { + fsync((*ppFile)->fd); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + } (*ppFile)->refId = 0; free(*ppFile); *ppFile = NULL; @@ -255,6 +273,7 @@ int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { if (pFile == NULL) { return 0; } + assert(pFile->fd >= 0); int64_t leftbytes = count; int64_t readbytes; char *tbuf = (char *)buf; @@ -282,10 +301,16 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) if (pFile == NULL) { return 0; } + assert(pFile->fd >= 0); return pread(pFile->fd, buf, count, offset); } int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + int64_t nleft = count; int64_t nwritten = 0; char *tbuf = (char *)buf; @@ -296,21 +321,20 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { if (errno == EINTR) { continue; } - fflush(pFile->fp); - fsync(pFile->fd); return -1; } nleft -= nwritten; tbuf += nwritten; } - - fflush(pFile->fp); fsync(pFile->fd); return count; } int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { - if (pFile == NULL) return -1; + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); return (int64_t)lseek(pFile->fd, (long)offset, whence); } @@ -318,6 +342,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); if (code < 0) { @@ -340,6 +369,11 @@ int32_t taosLockFile(TdFilePtr pFile) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); #endif } @@ -348,6 +382,11 @@ int32_t taosUnLockFile(TdFilePtr pFile) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); #endif } @@ -403,6 +442,11 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { return 0; #else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + return ftruncate(pFile->fd, l_size); #endif } @@ -419,7 +463,14 @@ int32_t taosFsyncFile(TdFilePtr pFile) { return FlushFileBuffers(h); #else - return fflush(pFile->fp); + if (pFile == NULL) { + return 0; + } + + if (pFile->fp != NULL) return fflush(pFile->fp); + if (pFile->fp >= 0) return fsync(pFile->fd); + + return 0; #endif } @@ -543,6 +594,11 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { + if (pFileSrc == NULL) { + return 0; + } + assert(pFileSrc->fd >= 0); + int64_t leftbytes = size; int64_t sentbytes; @@ -565,12 +621,22 @@ int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_ } int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; + } + assert(pFileOut->fd >= 0); + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } #endif void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + if (pFile == NULL) { + return; + } + assert(pFile->fp != NULL); + char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; va_list ap; va_start(ap, format); @@ -580,7 +646,10 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { } void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { - if (pFile == NULL) return NULL; + if (pFile == NULL) { + return NULL; + } + assert(pFile->fd >= 0); void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); return ptr; @@ -598,7 +667,19 @@ int32_t taosUmaskFile(int32_t maskVal) { int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { + if (pFile == NULL) { + return -1; + } + assert(pFile->fp != NULL); + size_t len = 0; return getline(ptrBuf, &len, pFile->fp); } -int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } \ No newline at end of file +int32_t taosEOFFile(TdFilePtr pFile) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fp != NULL); + + return feof(pFile->fp); +} \ No newline at end of file diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 45749588c11c1d5a5ec2c6c211c5ff056a4c4455..cf9c557f5ef5b36a23de42adf9b1a54e75a27f4c 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -404,7 +404,7 @@ bool taosGetSysMemory(float *memoryUsedMB) { bool taosGetProcMemory(float *memoryUsedMB) { // FILE *fp = fopen(tsProcMemFile, "r"); - TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; @@ -440,7 +440,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { // FILE *fp = fopen(tsSysCpuFile, "r"); - TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; @@ -465,7 +465,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { // FILE *fp = fopen(tsProcCpuFile, "r"); - TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; @@ -550,7 +550,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; // FILE *fp = fopen(tsSysNetFile, "r"); - TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; @@ -636,7 +636,7 @@ bool taosGetBandSpeed(float *bandSpeedKb) { bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { // FILE *fp = fopen(tsProcIOFile, "r"); - TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index 07846781adc1eaeb990740018977d4c68da34551..68da2ce25e1da8058dfeb1de60c6fb25c4f80ea3 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -46,6 +46,8 @@ #endif void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { + if (inTimezone == NULL || inTimezone[0] == 0) return; + #ifdef WINDOWS char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", inTimezone); diff --git a/source/util/src/exception.c b/source/util/src/exception.c deleted file mode 100644 index 9740b9031bd876e05fb9e38ec28722732904c7df..0000000000000000000000000000000000000000 --- a/source/util/src/exception.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "exception.h" - -static threadlocal SExceptionNode* expList; - -void exceptionPushNode( SExceptionNode* node ) { - node->prev = expList; - expList = node; -} - -int32_t exceptionPopNode() { - SExceptionNode* node = expList; - expList = node->prev; - return node->code; -} - -void exceptionThrow( int32_t code ) { - expList->code = code; - longjmp( expList->jb, 1 ); -} - - - -static void cleanupWrapper_void_ptr_ptr( SCleanupAction* ca ) { - void (*func)( void*, void* ) = ca->func; - func( ca->arg1.Ptr, ca->arg2.Ptr ); -} - -static void cleanupWrapper_void_ptr_bool( SCleanupAction* ca ) { - void (*func)( void*, bool ) = ca->func; - func( ca->arg1.Ptr, ca->arg2.Bool ); -} - -static void cleanupWrapper_void_ptr( SCleanupAction* ca ) { - void (*func)( void* ) = ca->func; - func( ca->arg1.Ptr ); -} - -static void cleanupWrapper_int_int( SCleanupAction* ca ) { - int (*func)( int ) = ca->func; - func( ca->arg1.Int ); -} - -static void cleanupWrapper_void( SCleanupAction* ca ) { - void (*func)() = ca->func; - func(); -} - -static void cleanupWrapper_int_ptr( SCleanupAction* ca ) { - int (*func)( void* ) = ca->func; - func( ca->arg1.Ptr ); -} - -typedef void (*wrapper)(SCleanupAction*); -static wrapper wrappers[] = { - cleanupWrapper_void_ptr_ptr, - cleanupWrapper_void_ptr_bool, - cleanupWrapper_void_ptr, - cleanupWrapper_int_int, - cleanupWrapper_void, - cleanupWrapper_int_ptr, -}; - - -void cleanupPush_void_ptr_ptr( bool failOnly, void* func, void* arg1, void* arg2 ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 0; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg1; - ca->arg2.Ptr = arg2; -} - -void cleanupPush_void_ptr_bool( bool failOnly, void* func, void* arg1, bool arg2 ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 1; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg1; - ca->arg2.Bool = arg2; -} - -void cleanupPush_void_ptr( bool failOnly, void* func, void* arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 2; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg; -} - -void cleanupPush_int_int( bool failOnly, void* func, int arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 3; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Int = arg; -} - -void cleanupPush_void( bool failOnly, void* func ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 4; - ca->failOnly = failOnly; - ca->func = func; -} - -void cleanupPush_int_ptr( bool failOnly, void* func, void* arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 5; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg; -} - - -int32_t cleanupGetActionCount() { - return expList->numCleanupAction; -} - - -static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed ) { - while( node->numCleanupAction > anchor ) { - --node->numCleanupAction; - SCleanupAction *ca = node->cleanupActions + node->numCleanupAction; - if( failed || !(ca->failOnly) ) { - wrappers[ca->wrapper]( ca ); - } - } -} - -void cleanupExecuteTo( int32_t anchor, bool failed ) { - doExecuteCleanup( expList, anchor, failed ); -} - -void cleanupExecute( SExceptionNode* node, bool failed ) { - doExecuteCleanup( node, 0, failed ); -} -bool cleanupExceedLimit() { - return expList->numCleanupAction >= expList->maxCleanupAction; -} diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index f8520b0742951cb6f63b6fc626326578282791e3..cb0a2d25a90a8999399e2e3bbfc3b4a4c838b0bd 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -13,30 +13,33 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "talgo.h" -#define doswap(__left, __right, __size, __buf) do {\ - memcpy((__buf), (__left), (__size));\ - memcpy((__left), (__right),(__size));\ - memcpy((__right), (__buf), (__size));\ -} while (0); +#define doswap(__left, __right, __size, __buf) \ + do { \ + memcpy((__buf), (__left), (__size)); \ + memcpy((__left), (__right), (__size)); \ + memcpy((__right), (__buf), (__size)); \ + } while (0); -static void median(void *src, size_t size, size_t s, size_t e, const void *param, __ext_compar_fn_t comparFn, void* buf) { +static void median(void *src, int64_t size, int64_t s, int64_t e, const void *param, __ext_compar_fn_t comparFn, + void *buf) { int32_t mid = ((int32_t)(e - s) >> 1u) + (int32_t)s; - + if (comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) == 1) { doswap(elePtrAt(src, size, mid), elePtrAt(src, size, s), size, buf); } - + if (comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, e), param) == 1) { doswap(elePtrAt(src, size, mid), elePtrAt(src, size, s), size, buf); doswap(elePtrAt(src, size, mid), elePtrAt(src, size, e), size, buf); } else if (comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) == 1) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } - - assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); + + assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && + comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW // tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); @@ -45,8 +48,8 @@ static void median(void *src, size_t size, size_t s, size_t e, const void *param #endif } -static void tInsertSort(void *src, size_t size, int32_t s, int32_t e, const void *param, __ext_compar_fn_t comparFn, - void* buf) { +static void tInsertSort(void *src, int64_t size, int32_t s, int32_t e, const void *param, __ext_compar_fn_t comparFn, + void *buf) { for (int32_t i = s + 1; i <= e; ++i) { for (int32_t j = i; j > s; --j) { if (comparFn(elePtrAt(src, size, j), elePtrAt(src, size, j - 1), param) == -1) { @@ -58,117 +61,117 @@ static void tInsertSort(void *src, size_t size, int32_t s, int32_t e, const void } } -static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const void *param, __ext_compar_fn_t comparFn, - void* buf) { +static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, const void *param, + __ext_compar_fn_t comparFn, void *buf) { // short array sort, incur another sort procedure instead of quick sort process const int32_t THRESHOLD_SIZE = 6; if (end - start + 1 <= THRESHOLD_SIZE) { tInsertSort(src, size, start, end, param, comparFn, buf); return; } - + median(src, size, start, end, param, comparFn, buf); - + int32_t s = start, e = end; int32_t endRightS = end, startLeftS = start; - + while (s < e) { while (e > s) { int32_t ret = comparFn(elePtrAt(src, size, e), elePtrAt(src, size, s), param); if (ret < 0) { break; } - - //move the data that equals to pivotal value to the right end of the list + + // move the data that equals to pivotal value to the right end of the list if (ret == 0 && e != endRightS) { doswap(elePtrAt(src, size, e), elePtrAt(src, size, endRightS), size, buf); endRightS--; } - + e--; } - + if (e != s) { doswap(elePtrAt(src, size, e), elePtrAt(src, size, s), size, buf); } - + while (s < e) { int32_t ret = comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param); if (ret > 0) { break; } - + if (ret == 0 && s != startLeftS) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, startLeftS), size, buf); startLeftS++; } s++; } - + if (e != s) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } } - + int32_t rightPartStart = e + 1; if (endRightS != end && e < end) { int32_t left = rightPartStart; int32_t right = end; - + while (right > endRightS && left <= endRightS) { doswap(elePtrAt(src, size, left), elePtrAt(src, size, right), size, buf); - + left++; right--; } - + rightPartStart += (end - endRightS); } - + int32_t leftPartEnd = e - 1; if (startLeftS != end && s > start) { int32_t left = start; int32_t right = leftPartEnd; - + while (left < startLeftS && right >= startLeftS) { doswap(elePtrAt(src, size, left), elePtrAt(src, size, right), size, buf); - + left++; right--; } - + leftPartEnd -= (startLeftS - start); } - + if (leftPartEnd > start) { tqsortImpl(src, start, leftPartEnd, size, param, comparFn, buf); } - + if (rightPartStart < end) { tqsortImpl(src, rightPartStart, end, size, param, comparFn, buf); } } -void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) { - char *buf = calloc(1, size); // prepare the swap buffer +void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { + char *buf = calloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); tfree(buf); } -void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) { +void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { // TODO: need to check the correctness of this function - int l = 0; - int r = (int)nmemb; - int idx = 0; - int comparison; - + int32_t l = 0; + int32_t r = (int32_t)nmemb; + int32_t idx = 0; + int32_t comparison; + if (flags == TD_EQ) { return bsearch(key, base, nmemb, size, compar); } else if (flags == TD_GE) { if (nmemb <= 0) return NULL; if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; - + while (l < r) { idx = (l + r) / 2; comparison = (*compar)(key, elePtrAt(base, size, idx)); @@ -180,7 +183,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx); } } - + if ((*compar)(key, elePtrAt(base, size, idx)) < 0) { return elePtrAt(base, size, idx); } else { @@ -194,7 +197,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, if (nmemb <= 0) return NULL; if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; - + while (l < r) { idx = (l + r) / 2; comparison = (*compar)(key, elePtrAt(base, size, idx)); @@ -206,7 +209,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx); } } - + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { return elePtrAt(base, size, idx); } else { @@ -216,20 +219,20 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx - 1); } } - + } else { assert(0); return NULL; } - + return NULL; } -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) -{ - int32_t parent; - int32_t child; - char *buf; +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + int32_t parent; + int32_t child; + char *buf; if (base && size > 0 && compar) { parent = start; @@ -244,7 +247,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const if (maxroot) { while (child <= end) { - if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { + if (child + 1 <= end && + (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { child++; } @@ -263,7 +267,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } } else { while (child <= end) { - if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { + if (child + 1 <= end && + (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { child++; } @@ -288,9 +293,9 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } } -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) -{ - int32_t i; +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + int32_t i; if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { @@ -298,15 +303,14 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, } } -/* - char *buf = calloc(1, size); + /* + char *buf = calloc(1, size); - for (i = len - 1; i > 0; i--) { - doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); - taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); - } + for (i = len - 1; i > 0; i--) { + doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); + taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); + } - tfree(buf); -*/ + tfree(buf); + */ } - diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 3e6f97c0c3bcdc30d39f08dd469babd3199a1fa9..1127a212559ab832a6afdb367c8ab5d4b46a6877 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tarray.h" -#include "talgo.h" SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); @@ -54,7 +53,7 @@ static int32_t taosArrayResize(SArray* pArray) { pArray->pData = tmp; pArray->capacity = size; - + return 0; } @@ -75,12 +74,12 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { return 0; } -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { +void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) { if (pArray == NULL || pData == NULL) { return NULL; } - if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){ + if (taosArrayEnsureCap(pArray, pArray->size + nEles) != 0) { return NULL; } @@ -91,7 +90,7 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { return dst; } -void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems) { +void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems) { assert(pArray != NULL && pData != NULL); if (numOfElems <= 0) { return; @@ -104,7 +103,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle } int32_t i = pData[0] + 1, j = 0; - while(i < size) { + while (i < size) { if (j == numOfElems - 1) { break; } @@ -133,7 +132,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle pArray->size -= numOfElems; } -void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)) { +void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) { assert(pArray); size_t size = pArray->size; @@ -142,7 +141,7 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } int32_t pos = 0; - for(int32_t i = 1; i < size; ++i) { + for (int32_t i = 1; i < size; ++i) { char* p1 = taosArrayGet(pArray, pos); char* p2 = taosArrayGet(pArray, i); @@ -164,7 +163,7 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } if (fp != NULL) { - for(int32_t i = pos + 1; i < pArray->size; ++i) { + for (int32_t i = pos + 1; i < pArray->size; ++i) { void* p = taosArrayGet(pArray, i); fp(p); } @@ -174,11 +173,11 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { - return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); + return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput)); } void* taosArrayPop(SArray* pArray) { - assert( pArray != NULL ); + assert(pArray != NULL); if (pArray->size == 0) { return NULL; @@ -194,15 +193,13 @@ void* taosArrayGet(const SArray* pArray, size_t index) { void* taosArrayGetP(const SArray* pArray, size_t index) { assert(index < pArray->size); - + void* d = TARRAY_GET_ELEM(pArray, index); - + return *(void**)d; } -void* taosArrayGetLast(const SArray* pArray) { - return TARRAY_GET_ELEM(pArray, pArray->size - 1); -} +void* taosArrayGetLast(const SArray* pArray) { return TARRAY_GET_ELEM(pArray, pArray->size - 1); } size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { @@ -227,7 +224,7 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { if (pArray->size >= pArray->capacity) { int32_t ret = taosArrayResize(pArray); - + if (ret < 0) { return NULL; } @@ -240,7 +237,7 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { memcpy(dst, pData, pArray->elemSize); pArray->size += 1; - + return dst; } @@ -252,7 +249,7 @@ void taosArraySet(SArray* pArray, size_t index, void* pData) { void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { assert(cnt <= pArray->size); pArray->size = pArray->size - cnt; - if(pArray->size == 0) { + if (pArray->size == 0) { return; } memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size * pArray->elemSize); @@ -265,14 +262,15 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); - + if (index == pArray->size - 1) { taosArrayPop(pArray); return; } - + size_t remain = pArray->size - index - 1; - memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + 1) * pArray->elemSize, + remain * pArray->elemSize); pArray->size -= 1; } @@ -288,13 +286,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { SArray* taosArrayDup(const SArray* pSrc) { assert(pSrc != NULL); - - if (pSrc->size == 0) { // empty array list + + if (pSrc->size == 0) { // empty array list return taosArrayInit(8, pSrc->elemSize); } - + SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize); - + memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); dst->size = pSrc->size; return dst; @@ -324,7 +322,7 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { return; } - for(int32_t i = 0; i < pArray->size; ++i) { + for (int32_t i = 0; i < pArray->size; ++i) { fp(TARRAY_GET_ELEM(pArray, i)); } @@ -338,14 +336,14 @@ void taosArraySort(SArray* pArray, __compar_fn_t compar) { qsort(pArray->pData, pArray->size, pArray->elemSize, compar); } -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) { assert(pArray != NULL && comparFn != NULL); assert(key != NULL); return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } -int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) { void* item = taosArraySearch(pArray, key, comparFn, flags); return item == NULL ? -1 : (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; } @@ -355,7 +353,7 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); } -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) { +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags) { assert(pArray != NULL); assert(key != NULL); @@ -366,47 +364,53 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t return *(char**)p; } -static int taosArrayPartition(SArray *pArray, int i, int j, __ext_compar_fn_t fn, const void *userData) { - void* key = taosArrayGetP(pArray, i); +static int32_t taosArrayPartition(SArray* pArray, int32_t i, int32_t j, __ext_compar_fn_t fn, const void* userData) { + void* key = taosArrayGetP(pArray, i); while (i < j) { - while (i < j && fn(taosArrayGetP(pArray, j), key, userData) >= 0) { j--; } - if (i < j) { - void *a = taosArrayGetP(pArray, j); + while (i < j && fn(taosArrayGetP(pArray, j), key, userData) >= 0) { + j--; + } + if (i < j) { + void* a = taosArrayGetP(pArray, j); taosArraySet(pArray, i, &a); } - while (i < j && fn(taosArrayGetP(pArray, i), key, userData) <= 0) { i++;} + while (i < j && fn(taosArrayGetP(pArray, i), key, userData) <= 0) { + i++; + } if (i < j) { - void *a = taosArrayGetP(pArray, i); + void* a = taosArrayGetP(pArray, i); taosArraySet(pArray, j, &a); } } - taosArraySet(pArray, i, &key); + taosArraySet(pArray, i, &key); return i; } -static void taosArrayQuicksortHelper(SArray *pArray, int low, int high, __ext_compar_fn_t fn, const void *param) { +static void taosArrayQuicksortHelper(SArray* pArray, int32_t low, int32_t high, __ext_compar_fn_t fn, + const void* param) { if (low < high) { - int idx = taosArrayPartition(pArray, low, high, fn, param); + int32_t idx = taosArrayPartition(pArray, low, high, fn, param); taosArrayQuicksortHelper(pArray, low, idx - 1, fn, param); taosArrayQuicksortHelper(pArray, idx + 1, high, fn, param); - } + } } -static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { +static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void* param) { if (pArray->size <= 1) { return; - } - taosArrayQuicksortHelper(pArray, 0, (int)(taosArrayGetSize(pArray) - 1), fn, param); + } + taosArrayQuicksortHelper(pArray, 0, (int32_t)(taosArrayGetSize(pArray) - 1), fn, param); } -static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + +static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void* param) { if (pArray->size <= 1) { return; - } - for (int i = 1; i <= pArray->size - 1; ++i) { - for (int j = i; j > 0; --j) { + } + for (int32_t i = 1; i <= pArray->size - 1; ++i) { + for (int32_t j = i; j > 0; --j) { if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { - void *a = taosArrayGetP(pArray, j); - void *b = taosArrayGetP(pArray, j - 1); + void* a = taosArrayGetP(pArray, j); + void* b = taosArrayGetP(pArray, j - 1); taosArraySet(pArray, j - 1, &a); taosArraySet(pArray, j, &b); } else { @@ -415,11 +419,10 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } } return; - -} -// order array -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - taosArrayGetSize(pArray) > 8 ? - taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); -} -//TODO(yihaoDeng) add order array +} + +// order array +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { + taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); +} +// TODO(yihaoDeng) add order array diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index 1b1f53df17d83313744d0ea528bfb013878eb53d..c5119e8b2d43cce3e0cc0781c54206da8f9f0bea 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -13,15 +13,15 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE +#include "tbase64.h" -// deprecated this file for bug prone -// base64 encode static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -char *base64_encode(const unsigned char *value, int vlen) { - unsigned char oval = 0; - char * result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); - char * out = result; + +char *base64_encode(const uint8_t *value, int32_t vlen) { + uint8_t oval = 0; + char *result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); + char *out = result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; *out++ = basis_64[((value[0] << 4) & 0x30) | (value[1] >> 4)]; @@ -42,8 +42,8 @@ char *base64_encode(const unsigned char *value, int vlen) { return result; } -// base64 decode #define CHAR64(c) (((c) < 0 || (c) > 127) ? -1 : index_64[(c)]) + static signed char index_64[128] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54, 55, @@ -51,10 +51,10 @@ static signed char index_64[128] = { 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1}; -unsigned char *base64_decode(const char *value, int inlen, int *outlen) { - int c1, c2, c3, c4; - unsigned char *result = (unsigned char *)malloc((size_t)(inlen * 3) / 4 + 1); - unsigned char *out = result; +uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { + int32_t c1, c2, c3, c4; + uint8_t *result = (uint8_t *)malloc((size_t)(inlen * 3) / 4 + 1); + uint8_t *out = result; *outlen = 0; @@ -80,13 +80,13 @@ unsigned char *base64_decode(const char *value, int inlen, int *outlen) { if ((c4 != '=') && (CHAR64(c4) == -1)) goto base64_decode_error; value += 4; - *out++ = (unsigned char)((CHAR64(c1) << 2) | (CHAR64(c2) >> 4)); + *out++ = (uint8_t)((CHAR64(c1) << 2) | (CHAR64(c2) >> 4)); *outlen += 1; if (c3 != '=') { - *out++ = (unsigned char)(((CHAR64(c2) << 4) & 0xf0) | (CHAR64(c3) >> 2)); + *out++ = (uint8_t)(((CHAR64(c2) << 4) & 0xf0) | (CHAR64(c3) >> 2)); *outlen += 1; if (c4 != '=') { - *out++ = (unsigned char)(((CHAR64(c3) << 6) & 0xc0) | CHAR64(c4)); + *out++ = (uint8_t)(((CHAR64(c3) << 6) & 0xc0) | CHAR64(c4)); *outlen += 1; } } diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index 0456d6a2eecca33a9509767dda27406bdf887e54..dc8c4b70c5107358a742e16ceef67584a111ff69 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -14,11 +14,8 @@ */ #define _DEFAULT_SOURCE - #include "tbuffer.h" -#include "exception.h" -#include "os.h" -//#include "taoserror.h" +#include "texception.h" typedef union Un4B { uint32_t ui; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 6914e7e2d03603a2f511d6f10c94239eef80d40f..64d822f750837d2b60b64c927b46288b149588ba 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -14,11 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "os.h" +#include "tcache.h" #include "tlog.h" #include "ttimer.h" #include "tutil.h" -#include "tcache.h" static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) @@ -62,15 +61,15 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * @param handle Cache object handle */ -static void* taosCacheTimedRefresh(void *handle); +static void *taosCacheTimedRefresh(void *handle); -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; -static SArray* pCacheArrayList = NULL; -static bool stopRefreshWorker = false; -static bool refreshWorkerNormalStopped = false; -static bool refreshWorkerUnexpectedStopped = false; +static pthread_t cacheRefreshWorker = {0}; +static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static SArray *pCacheArrayList = NULL; +static bool stopRefreshWorker = false; +static bool refreshWorkerNormalStopped = false; +static bool refreshWorkerUnexpectedStopped = false; static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); @@ -83,7 +82,7 @@ static void doInitRefreshThread(void) { pthread_attr_destroy(&thattr); } -pthread_t doRegisterCacheObj(SCacheObj* pCacheObj) { +pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { pthread_once(&cacheThreadInit, doInitRefreshThread); pthread_mutex_lock(&guard); @@ -102,7 +101,8 @@ pthread_t doRegisterCacheObj(SCacheObj* pCacheObj) { * @param lifespan total survial expiredTime from now * @return SCacheDataNode */ -static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration); +static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, + uint64_t duration); /** * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash @@ -146,18 +146,18 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo free(pNode); } -static FORCE_INLINE STrashElem* doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) { - if (pElem->pData->signature != (uint64_t) pElem->pData) { +static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { + if (pElem->pData->signature != (uint64_t)pElem->pData) { uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); return NULL; } - STrashElem* next = pElem->next; + STrashElem *next = pElem->next; pCacheObj->numOfElemsInTrash--; if (pElem->prev) { pElem->prev->next = pElem->next; - } else { // pnode is the header, update header + } else { // pnode is the header, update header pCacheObj->pTrash = pElem->next; } @@ -172,7 +172,7 @@ static FORCE_INLINE STrashElem* doRemoveElemInTrashcan(SCacheObj* pCacheObj, STr return next; } -static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem *pElem) { +static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem *pElem) { if (pCacheObj->freeFp) { pCacheObj->freeFp(pElem->pData->data); } @@ -181,19 +181,20 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem free(pElem); } -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char* cacheName) { - const int32_t SLEEP_DURATION = 500; //500 ms +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, + const char *cacheName) { + const int32_t SLEEP_DURATION = 500; // 500 ms if (refreshTimeInSeconds <= 0) { return NULL; } - + SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - + pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { @@ -201,17 +202,17 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - + // set free cache node callback function - pCacheObj->freeFp = fn; + pCacheObj->freeFp = fn; pCacheObj->refreshTime = refreshTimeInSeconds * 1000; - pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; + pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__cache_lock_init(pCacheObj) != 0) { taosHashCleanup(pCacheObj->pHashTable); free(pCacheObj); - + uError("failed to init lock, reason:%s", strerror(errno)); return NULL; } @@ -220,7 +221,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, + int32_t durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } @@ -242,8 +244,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); } else { // duplicated key exists while (1) { - SCacheDataNode* p = NULL; -// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); + SCacheDataNode *p = NULL; + // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen); // add to trashcan @@ -283,10 +285,10 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return pNode1->data; } -static void incRefFn(void* ptNode) { +static void incRefFn(void *ptNode) { assert(ptNode != NULL); - SCacheDataNode** p = (SCacheDataNode**) ptNode; + SCacheDataNode **p = (SCacheDataNode **)ptNode; assert(T_REF_VAL_GET(*p) >= 0); int32_t ret = T_REF_INC(*p); @@ -303,15 +305,16 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return NULL; } - SCacheDataNode* ptNode = NULL; + SCacheDataNode *ptNode = NULL; taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode); -// taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); + // taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); - void* pData = (ptNode != NULL)? ptNode->data:NULL; + void *pData = (ptNode != NULL) ? ptNode->data : NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(ptNode)); + uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, + T_REF_VAL_GET(ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -323,10 +326,10 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; - + size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset); - + if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; @@ -342,22 +345,22 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; - + size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); - + if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } - + assert(T_REF_VAL_GET(ptNode) >= 1); - + char *d = *data; - + // clear its reference to old area *data = NULL; - + return d; } @@ -371,13 +374,12 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { return; } - // The operation of removal from hash table and addition to trashcan is not an atomic operation, // therefore the check for the empty of both the hash table and the trashcan has a race condition. // It happens when there is only one object in the cache, and two threads which has referenced this object // start to free the it simultaneously [TD-1569]. size_t offset = offsetof(SCacheDataNode, data); - + SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); if (pNode->signature != (uint64_t)pNode) { uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode); @@ -391,20 +393,20 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (pCacheObj->extendLifespan && (!inTrashcan) && (!_remove)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); - uDebug("cache:%s, data:%p extend expire time: %"PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); } if (_remove) { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - char* key = pNode->key; - char* d = pNode->data; + char *key = pNode->key; + char *d = pNode->data; int32_t ref = T_REF_VAL_GET(pNode); uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, key, d, ref - 1, inTrashcan); /* - * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users - * releasing this resources. + * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all + * users releasing this resources. * * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread * that tries to do the same thing. @@ -433,16 +435,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. SCacheDataNode *p = NULL; - int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); -// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void *)); + int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void + // *)); ref = T_REF_DEC(pNode); // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. // note that the remove operation can be executed only once. if (ret == 0) { if (p != pNode) { - uDebug( "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " - "others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); + uDebug( + "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " + "others already", + pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); assert(p->pTNodeHeader == NULL); taosAddToTrashcan(pCacheObj, p); @@ -468,35 +473,36 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } } } else { - uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", - pCacheObj->name, pNode->key, pNode->data, ref); + uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name, + pNode->key, pNode->data, ref); } } } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - char* key = pNode->key; - char* p = pNode->data; - -// int32_t ref = T_REF_VAL_GET(pNode); -// -// if (ref == 1 && inTrashcan) { -// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be -// // destroyed by refresh worker if decrease ref count before removing it from linked-list. -// assert(pNode->pTNodeHeader->pData == pNode); -// -// __cache_wr_lock(pCacheObj); -// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); -// __cache_unlock(pCacheObj); -// -// ref = T_REF_DEC(pNode); -// assert(ref == 0); -// -// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); -// } else { -// ref = T_REF_DEC(pNode); -// assert(ref >= 0); -// } + char *key = pNode->key; + char *p = pNode->data; + + // int32_t ref = T_REF_VAL_GET(pNode); + // + // if (ref == 1 && inTrashcan) { + // // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may + // be + // // destroyed by refresh worker if decrease ref count before removing it from linked-list. + // assert(pNode->pTNodeHeader->pData == pNode); + // + // __cache_wr_lock(pCacheObj); + // doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); + // __cache_unlock(pCacheObj); + // + // ref = T_REF_DEC(pNode); + // assert(ref == 0); + // + // doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); + // } else { + // ref = T_REF_DEC(pNode); + // assert(ref >= 0); + // } int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); @@ -504,21 +510,21 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } typedef struct SHashTravSupp { - SCacheObj* pCacheObj; - int64_t time; + SCacheObj *pCacheObj; + int64_t time; __cache_trav_fn_t fp; - void* param1; + void *param1; } SHashTravSupp; -static bool travHashTableEmptyFn(void* param, void* data) { - SHashTravSupp* ps = (SHashTravSupp*) param; - SCacheObj* pCacheObj= ps->pCacheObj; +static bool travHashTableEmptyFn(void *param, void *data) { + SHashTravSupp *ps = (SHashTravSupp *)param; + SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode *pNode = *(SCacheDataNode **) data; + SCacheDataNode *pNode = *(SCacheDataNode **)data; if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); - } else { // do add to trashcan + } else { // do add to trashcan taosAddToTrashcan(pCacheObj, pNode); } @@ -529,7 +535,7 @@ static bool travHashTableEmptyFn(void* param, void* data) { void taosCacheEmpty(SCacheObj *pCacheObj) { SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); taosTrashcanEmpty(pCacheObj, false); } @@ -542,9 +548,9 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { // wait for the refresh thread quit before destroying the cache object. // But in the dll, the child thread will be killed before atexit takes effect. - while(atomic_load_8(&pCacheObj->deleting) != 0) { - if (refreshWorkerNormalStopped) break; - if (refreshWorkerUnexpectedStopped) return; + while (atomic_load_8(&pCacheObj->deleting) != 0) { + if (refreshWorkerNormalStopped) break; + if (refreshWorkerUnexpectedStopped) return; taosMsleep(50); } @@ -568,11 +574,11 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->lifespan = duration; - pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; + pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)totalSize; return pNewNode; } @@ -610,16 +616,17 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { if (pCacheObj->numOfElemsInTrash == 0) { if (pCacheObj->pTrash != NULL) { pCacheObj->pTrash = NULL; - uError("cache:%s, key:inconsistency data in cache, numOfElem in trashcan:%d", pCacheObj->name, pCacheObj->numOfElemsInTrash); + uError("cache:%s, key:inconsistency data in cache, numOfElem in trashcan:%d", pCacheObj->name, + pCacheObj->numOfElemsInTrash); } __cache_unlock(pCacheObj); return; } - const char* stat[] = {"false", "true"}; + const char *stat[] = {"false", "true"}; uDebug("cache:%s start to cleanup trashcan, numOfElem in trashcan:%d, free:%s", pCacheObj->name, - pCacheObj->numOfElemsInTrash, (force? stat[1]:stat[0])); + pCacheObj->numOfElemsInTrash, (force ? stat[1] : stat[0])); STrashElem *pElem = pCacheObj->pTrash; while (pElem) { @@ -627,8 +634,8 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { assert(pElem->next != pElem && pElem->prev != pElem); if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, pElem->pData->data, - pCacheObj->numOfElemsInTrash - 1); + uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, + pElem->pData->data, pCacheObj->numOfElemsInTrash - 1); doRemoveElemInTrashcan(pCacheObj, pElem); doDestroyTrashcanElem(pCacheObj, pElem); @@ -642,25 +649,25 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { } void doCleanupDataCache(SCacheObj *pCacheObj) { -// SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + // SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); taosTrashcanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); - + tfree(pCacheObj->name); memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } -bool travHashTableFn(void* param, void* data) { - SHashTravSupp* ps = (SHashTravSupp*) param; - SCacheObj* pCacheObj= ps->pCacheObj; +bool travHashTableFn(void *param, void *data) { + SHashTravSupp *ps = (SHashTravSupp *)param; + SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode* pNode = *(SCacheDataNode **) data; + SCacheDataNode *pNode = *(SCacheDataNode **)data; if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); @@ -676,30 +683,30 @@ bool travHashTableFn(void* param, void* data) { return true; } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_trav_fn_t fp, void* param1) { +static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { assert(pCacheObj != NULL); SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } void taosCacheRefreshWorkerUnexpectedStopped(void) { - if(!refreshWorkerNormalStopped) { - refreshWorkerUnexpectedStopped=true; + if (!refreshWorkerNormalStopped) { + refreshWorkerUnexpectedStopped = true; } } -void* taosCacheTimedRefresh(void *handle) { +void *taosCacheTimedRefresh(void *handle) { assert(pCacheArrayList != NULL); uDebug("cache refresh thread starts"); setThreadName("cacheRefresh"); - const int32_t SLEEP_DURATION = 500; //500 ms - int64_t count = 0; + const int32_t SLEEP_DURATION = 500; // 500 ms + int64_t count = 0; atexit(taosCacheRefreshWorkerUnexpectedStopped); - while(1) { + while (1) { taosMsleep(SLEEP_DURATION); if (stopRefreshWorker) { goto _end; @@ -711,9 +718,9 @@ void* taosCacheTimedRefresh(void *handle) { count += 1; - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { pthread_mutex_lock(&guard); - SCacheObj* pCacheObj = taosArrayGetP(pCacheArrayList, i); + SCacheObj *pCacheObj = taosArrayGetP(pCacheArrayList, i); if (pCacheObj == NULL) { uError("object is destroyed. ignore and try next"); @@ -726,8 +733,8 @@ void* taosCacheTimedRefresh(void *handle) { taosArrayRemove(pCacheArrayList, i); size = taosArrayGetSize(pCacheArrayList); - uDebug("%s is destroying, remove it from refresh list, remain cache obj:%"PRIzu, pCacheObj->name, size); - pCacheObj->deleting = 0; //reset the deleting flag to enable pCacheObj to continue releasing resources. + uDebug("%s is destroying, remove it from refresh list, remain cache obj:%" PRIzu, pCacheObj->name, size); + pCacheObj->deleting = 0; // reset the deleting flag to enable pCacheObj to continue releasing resources. pthread_mutex_unlock(&guard); continue; @@ -757,18 +764,18 @@ void* taosCacheTimedRefresh(void *handle) { } } - _end: +_end: taosArrayDestroy(pCacheArrayList); pCacheArrayList = NULL; pthread_mutex_destroy(&guard); - refreshWorkerNormalStopped=true; + refreshWorkerNormalStopped = true; uDebug("cache refresh thread quits"); return NULL; } -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) { +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) { if (pCacheObj == NULL) { return; } @@ -777,6 +784,4 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) doCacheRefresh(pCacheObj, now, fp, param1); } -void taosStopCacheRefreshWorker(void) { - stopRefreshWorker = true; -} \ No newline at end of file +void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; } \ No newline at end of file diff --git a/source/util/src/compare.c b/source/util/src/tcompare.c similarity index 65% rename from source/util/src/compare.c rename to source/util/src/tcompare.c index b597eb32bf9f3a8a186d827c93ee748c0b20f7c1..062d144933136891acc98b4ba07b43d09c8a9f69 100644 --- a/source/util/src/compare.c +++ b/source/util/src/tcompare.c @@ -16,15 +16,12 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE -#include "os.h" - -#include "compare.h" +#include "tcompare.h" #include "regex.h" +#include "tdef.h" #include "thash.h" -#include "types.h" #include "tlog.h" -#include "tdef.h" -#include "taos.h" +#include "types.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -58,16 +55,14 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } - -int32_t compareChkInString(const void *pLeft, const void* pRight) { +int32_t compareChkInString(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } -int32_t compareChkNotInString(const void *pLeft, const void* pRight) { +int32_t compareChkNotInString(const void *pLeft, const void *pRight) { return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } - int32_t compareInt8Val(const void *pLeft, const void *pRight) { int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; @@ -75,9 +70,7 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { - return compareInt8Val(pRight, pLeft); -} +int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { return compareInt8Val(pRight, pLeft); } int32_t compareInt16Val(const void *pLeft, const void *pRight) { int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); @@ -86,9 +79,7 @@ int32_t compareInt16Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) { - return compareInt16Val(pRight, pLeft); -} +int32_t compareInt16ValDesc(const void *pLeft, const void *pRight) { return compareInt16Val(pRight, pLeft); } int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); @@ -97,9 +88,7 @@ int32_t compareInt32Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) { - return compareInt32Val(pRight, pLeft); -} +int32_t compareInt32ValDesc(const void *pLeft, const void *pRight) { return compareInt32Val(pRight, pLeft); } int32_t compareInt64Val(const void *pLeft, const void *pRight) { int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); @@ -108,9 +97,7 @@ int32_t compareInt64Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) { - return compareInt64Val(pRight, pLeft); -} +int32_t compareInt64ValDesc(const void *pLeft, const void *pRight) { return compareInt64Val(pRight, pLeft); } int32_t compareUint32Val(const void *pLeft, const void *pRight) { uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); @@ -119,9 +106,7 @@ int32_t compareUint32Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) { - return compareUint32Val(pRight, pLeft); -} +int32_t compareUint32ValDesc(const void *pLeft, const void *pRight) { return compareUint32Val(pRight, pLeft); } int32_t compareUint64Val(const void *pLeft, const void *pRight) { uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); @@ -130,9 +115,7 @@ int32_t compareUint64Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) { - return compareUint64Val(pRight, pLeft); -} +int32_t compareUint64ValDesc(const void *pLeft, const void *pRight) { return compareUint64Val(pRight, pLeft); } int32_t compareUint16Val(const void *pLeft, const void *pRight) { uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); @@ -141,20 +124,16 @@ int32_t compareUint16Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) { - return compareUint16Val(pRight, pLeft); -} +int32_t compareUint16ValDesc(const void *pLeft, const void *pRight) { return compareUint16Val(pRight, pLeft); } -int32_t compareUint8Val(const void* pLeft, const void* pRight) { +int32_t compareUint8Val(const void *pLeft, const void *pRight) { uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } -int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) { - return compareUint8Val(pRight, pLeft); -} +int32_t compareUint8ValDesc(const void *pLeft, const void *pRight) { return compareUint8Val(pRight, pLeft); } int32_t compareFloatVal(const void *pLeft, const void *pRight) { float p1 = GET_FLOAT_VAL(pLeft); @@ -174,12 +153,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) { if (FLT_EQUAL(p1, p2)) { return 0; } - return FLT_GREATER(p1, p2) ? 1: -1; + return FLT_GREATER(p1, p2) ? 1 : -1; } -int32_t compareFloatValDesc(const void* pLeft, const void* pRight) { - return compareFloatVal(pRight, pLeft); -} +int32_t compareFloatValDesc(const void *pLeft, const void *pRight) { return compareFloatVal(pRight, pLeft); } int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double p1 = GET_DOUBLE_VAL(pLeft); @@ -199,30 +176,28 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { if (FLT_EQUAL(p1, p2)) { return 0; } - return FLT_GREATER(p1, p2) ? 1: -1; + return FLT_GREATER(p1, p2) ? 1 : -1; } -int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) { - return compareDoubleVal(pRight, pLeft); -} +int32_t compareDoubleValDesc(const void *pLeft, const void *pRight) { return compareDoubleVal(pRight, pLeft); } int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); if (len1 != len2) { - return len1 > len2? 1:-1; + return len1 > len2 ? 1 : -1; } else { int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1); if (ret == 0) { return 0; } else { - return ret > 0 ? 1:-1; + return ret > 0 ? 1 : -1; } } } -int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) { +int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedStr(pRight, pLeft); } @@ -231,9 +206,9 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len2 = varDataLen(pRight); if (len1 != len2) { - return len1 > len2? 1:-1; + return len1 > len2 ? 1 : -1; } else { - int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1); + int32_t ret = memcmp((wchar_t *)pLeft, (wchar_t *)pRight, len1); if (ret == 0) { return 0; } else { @@ -242,7 +217,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { } } -int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { +int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } @@ -302,10 +277,13 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const } c1 = str[j++]; - ++o; - + ++o; + if (j <= size) { - if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } + if (c == '\\' && patterStr[i] == '_' && c1 == '_') { + i++; + continue; + } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } @@ -365,21 +343,19 @@ int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t siz return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) { - return compareStrRegexComp(pLeft, pRight); -} +int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); } -int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) { +int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight) ? 0 : 1; } -int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { +int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); + char *pattern = malloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); pattern[sz] = 0; @@ -414,15 +390,14 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { return result; } -int32_t taosArrayCompareString(const void* a, const void* b) { - const char* x = *(const char**)a; - const char* y = *(const char**)b; +int32_t taosArrayCompareString(const void *a, const void *b) { + const char *x = *(const char **)a; + const char *y = *(const char **)b; return compareLenPrefixedStr(x, y); } - -int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { +int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); @@ -430,7 +405,7 @@ int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); + char *buf = malloc(sz + 1); memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; @@ -440,11 +415,11 @@ int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { +int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) { return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; } -int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { +int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); @@ -452,17 +427,16 @@ int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { +int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight) { return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; } - __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; @@ -514,13 +488,25 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { switch (type) { case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; - case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_TINYINT: + comparFn = compareInt8Val; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = compareInt16Val; + break; + case TSDB_DATA_TYPE_INT: + comparFn = compareInt32Val; + break; case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = compareInt64Val; + break; + case TSDB_DATA_TYPE_FLOAT: + comparFn = compareFloatVal; + break; + case TSDB_DATA_TYPE_DOUBLE: + comparFn = compareDoubleVal; + break; case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; @@ -560,10 +546,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { break; } - case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; - case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + case TSDB_DATA_TYPE_UTINYINT: + comparFn = compareUint8Val; + break; + case TSDB_DATA_TYPE_USMALLINT: + comparFn = compareUint16Val; + break; + case TSDB_DATA_TYPE_UINT: + comparFn = compareUint32Val; + break; + case TSDB_DATA_TYPE_UBIGINT: + comparFn = compareUint64Val; + break; default: comparFn = compareInt32Val; @@ -573,4 +567,106 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { return comparFn; } +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { + __compar_fn_t comparFn = NULL; + switch (keyType) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt8Val : compareInt8ValDesc; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt16Val : compareInt16ValDesc; + break; + case TSDB_DATA_TYPE_INT: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; + break; + case TSDB_DATA_TYPE_FLOAT: + comparFn = (order == TSDB_ORDER_ASC) ? compareFloatVal : compareFloatValDesc; + break; + case TSDB_DATA_TYPE_DOUBLE: + comparFn = (order == TSDB_ORDER_ASC) ? compareDoubleVal : compareDoubleValDesc; + break; + case TSDB_DATA_TYPE_UTINYINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint8Val : compareUint8ValDesc; + break; + case TSDB_DATA_TYPE_USMALLINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint16Val : compareUint16ValDesc; + break; + case TSDB_DATA_TYPE_UINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc; + break; + case TSDB_DATA_TYPE_UBIGINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; + break; + case TSDB_DATA_TYPE_BINARY: + comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; + break; + case TSDB_DATA_TYPE_NCHAR: + comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; + break; + default: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; + break; + } + + return comparFn; +} + +int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { + switch (type) { + case TSDB_DATA_TYPE_INT: + DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: + DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_BIGINT: + DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); + case TSDB_DATA_TYPE_SMALLINT: + DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); + case TSDB_DATA_TYPE_UTINYINT: + DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); + case TSDB_DATA_TYPE_USMALLINT: + DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); + case TSDB_DATA_TYPE_UINT: + DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); + case TSDB_DATA_TYPE_UBIGINT: + DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); + case TSDB_DATA_TYPE_NCHAR: { + tstr *t1 = (tstr *)f1; + tstr *t2 = (tstr *)f2; + + if (t1->len != t2->len) { + return t1->len > t2->len ? 1 : -1; + } + int32_t ret = memcmp((wchar_t *)t1, (wchar_t *)t2, t2->len); + if (ret == 0) { + return ret; + } + return (ret < 0) ? -1 : 1; + } + default: { // todo refactor + tstr *t1 = (tstr *)f1; + tstr *t2 = (tstr *)f2; + + if (t1->len != t2->len) { + return t1->len > t2->len ? 1 : -1; + } else { + int32_t ret = strncmp(t1->data, t2->data, t1->len); + if (ret == 0) { + return 0; + } else { + return ret < 0 ? -1 : 1; + } + } + } + } +} diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 03a4846c3de75f6b928320203759ff492beb3dae..b2332aded7fd79beb8feb6539ddf8a1200de4a5b 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -16,7 +16,7 @@ /* README.md TAOS compression * * INTEGER Compression Algorithm: - * To compress integers (including char, short, int, int64_t), the difference + * To compress integers (including char, short, int32_t, int64_t), the difference * between two integers is calculated at first. Then the difference is * transformed to positive by zig-zag encoding method * (https://gist.github.com/mfuerstenau/ba870a29e16536fdbaba). Then the value is @@ -47,69 +47,65 @@ * */ -#include "os.h" -#include "lz4.h" -#ifdef TD_TSZ - #include "td_sz.h" -#endif +#define _DEFAULT_SOURCE #include "tcompression.h" +#include "lz4.h" #include "tlog.h" -static const int TEST_NUMBER = 1; -#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) +#ifdef TD_TSZ +#include "td_sz.h" +#endif + +static const int32_t TEST_NUMBER = 1; +#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) #define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L) -#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) +#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #ifdef TD_TSZ -bool lossyFloat = false; +bool lossyFloat = false; bool lossyDouble = false; // init call -int tsCompressInit(){ - // config - if(lossyColumns[0] == 0){ +int32_t tsCompressInit() { + // config + if (lossyColumns[0] == 0) { lossyFloat = false; lossyDouble = false; return 0; } - lossyFloat = strstr(lossyColumns, "float") != NULL; + lossyFloat = strstr(lossyColumns, "float") != NULL; lossyDouble = strstr(lossyColumns, "double") != NULL; - if(lossyFloat == false && lossyDouble == false) - return 0; - + if (lossyFloat == false && lossyDouble == false) return 0; + tdszInit(fPrecision, dPrecision, maxRange, curRange, Compressor); - if(lossyFloat) - uInfo("lossy compression float is opened. "); - if(lossyDouble) - uInfo("lossy compression double is opened. "); + if (lossyFloat) uInfo("lossy compression float is opened. "); + if (lossyDouble) uInfo("lossy compression double is opened. "); return 1; } // exit call -void tsCompressExit(){ - tdszExit(); -} +void tsCompressExit() { tdszExit(); } #endif /* * Compress Integer (Simple8B). */ -int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type) { +int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 // 12 13 14 15 - char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; - int selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; - char bit_to_selector[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, + char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + char bit_to_selector[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; // get the byte limit. - int word_length = 0; + int32_t word_length = 0; switch (type) { case TSDB_DATA_TYPE_BIGINT: word_length = LONG_BYTES; @@ -128,17 +124,17 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o return -1; } - int byte_limit = nelements * word_length + 1; - int opos = 1; + int32_t byte_limit = nelements * word_length + 1; + int32_t opos = 1; int64_t prev_value = 0; - for (int i = 0; i < nelements;) { + for (int32_t i = 0; i < nelements;) { char selector = 0; char bit = 0; - int elems = 0; + int32_t elems = 0; int64_t prev_value_tmp = prev_value; - for (int j = i; j < nelements; j++) { + for (int32_t j = i; j < nelements; j++) { // Read data from the input stream and convert it to INT64 type. int64_t curr_value = 0; switch (type) { @@ -172,16 +168,17 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o tmp_bit = (LONG_BYTES * BITS_PER_BYTE) - BUILDIN_CLZL(zigzag_value); } - if (elems + 1 <= selector_to_elems[(int)selector] && elems + 1 <= selector_to_elems[(int)(bit_to_selector[(int)tmp_bit])]) { + if (elems + 1 <= selector_to_elems[(int32_t)selector] && + elems + 1 <= selector_to_elems[(int32_t)(bit_to_selector[(int32_t)tmp_bit])]) { // If can hold another one. - selector = selector > bit_to_selector[(int)tmp_bit] ? selector : bit_to_selector[(int)tmp_bit]; + selector = selector > bit_to_selector[(int32_t)tmp_bit] ? selector : bit_to_selector[(int32_t)tmp_bit]; elems++; - bit = bit_per_integer[(int)selector]; + bit = bit_per_integer[(int32_t)selector]; } else { // if cannot hold another one. - while (elems < selector_to_elems[(int)selector]) selector++; - elems = selector_to_elems[(int)selector]; - bit = bit_per_integer[(int)selector]; + while (elems < selector_to_elems[(int32_t)selector]) selector++; + elems = selector_to_elems[(int32_t)selector]; + bit = bit_per_integer[(int32_t)selector]; break; } prev_value_tmp = curr_value; @@ -189,7 +186,7 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o uint64_t buffer = 0; buffer |= (uint64_t)selector; - for (int k = 0; k < elems; k++) { + for (int32_t k = 0; k < elems; k++) { int64_t curr_value = 0; /* get current values */ switch (type) { case TSDB_DATA_TYPE_TINYINT: @@ -229,8 +226,8 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o return opos; } -int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type) { - int word_length = 0; +int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { + int32_t word_length = 0; switch (type) { case TSDB_DATA_TYPE_BIGINT: word_length = LONG_BYTES; @@ -257,12 +254,12 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 // 12 13 14 15 - char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; - int selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; const char *ip = input + 1; - int count = 0; - int _pos = 0; + int32_t count = 0; + int32_t _pos = 0; int64_t prev_value = 0; while (1) { @@ -271,11 +268,11 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const uint64_t w = 0; memcpy(&w, ip, LONG_BYTES); - char selector = (char)(w & INT64MASK(4)); // selector = 4 - char bit = bit_per_integer[(int)selector]; // bit = 3 - int elems = selector_to_elems[(int)selector]; + char selector = (char)(w & INT64MASK(4)); // selector = 4 + char bit = bit_per_integer[(int32_t)selector]; // bit = 3 + int32_t elems = selector_to_elems[(int32_t)selector]; - for (int i = 0; i < elems; i++) { + for (int32_t i = 0; i < elems; i++) { uint64_t zigzag_value; if (selector == 0 || selector == 1) { @@ -320,11 +317,11 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const /* ----------------------------------------------Bool Compression * ---------------------------------------------- */ // TODO: You can also implement it using RLE method. -int tsCompressBoolImp(const char *const input, const int nelements, char *const output) { - int pos = -1; - int ele_per_byte = BITS_PER_BYTE / 2; +int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output) { + int32_t pos = -1; + int32_t ele_per_byte = BITS_PER_BYTE / 2; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % ele_per_byte == 0) { pos++; output[pos] = 0; @@ -351,11 +348,11 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const return pos + 1; } -int tsDecompressBoolImp(const char *const input, const int nelements, char *const output) { - int ipos = -1, opos = 0; - int ele_per_byte = BITS_PER_BYTE / 2; +int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output) { + int32_t ipos = -1, opos = 0; + int32_t ele_per_byte = BITS_PER_BYTE / 2; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % ele_per_byte == 0) { ipos++; } @@ -374,10 +371,10 @@ int tsDecompressBoolImp(const char *const input, const int nelements, char *cons } /* Run Length Encoding(RLE) Method */ -int tsCompressBoolRLEImp(const char *const input, const int nelements, char *const output) { - int _pos = 0; +int32_t tsCompressBoolRLEImp(const char *const input, const int32_t nelements, char *const output) { + int32_t _pos = 0; - for (int i = 0; i < nelements;) { + for (int32_t i = 0; i < nelements;) { unsigned char counter = 1; char num = input[i]; @@ -407,8 +404,8 @@ int tsCompressBoolRLEImp(const char *const input, const int nelements, char *con return _pos; } -int tsDecompressBoolRLEImp(const char *const input, const int nelements, char *const output) { - int ipos = 0, opos = 0; +int32_t tsDecompressBoolRLEImp(const char *const input, const int32_t nelements, char *const output) { + int32_t ipos = 0, opos = 0; while (1) { char encode = input[ipos++]; unsigned counter = (encode >> 1) & INT8MASK(7); @@ -427,9 +424,9 @@ int tsDecompressBoolRLEImp(const char *const input, const int nelements, char *c // Note: the size of the output must be larger than input_size + 1 and // LZ4_compressBound(size) + 1; // >= max(input_size, LZ4_compressBound(input_size)) + 1; -int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize) { +int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize) { // Try to compress using LZ4 algorithm. - const int compressed_data_size = LZ4_compress_default(input, output + 1, inputSize, outputSize-1); + const int32_t compressed_data_size = LZ4_compress_default(input, output + 1, inputSize, outputSize - 1); // If cannot compress or after compression, data becomes larger. if (compressed_data_size <= 0 || compressed_data_size > inputSize) { @@ -443,12 +440,12 @@ int tsCompressStringImp(const char *const input, int inputSize, char *const outp return compressed_data_size + 1; } -int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize) { +int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, int32_t outputSize) { // compressedSize is the size of data after compression. - + if (input[0] == 1) { /* It is compressed by LZ4 algorithm */ - const int decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); + const int32_t decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); if (decompressed_size < 0) { uError("Failed to decompress string with LZ4 algorithm, decompressed size:%d", decompressed_size); return -1; @@ -468,24 +465,24 @@ int tsDecompressStringImp(const char *const input, int compressedSize, char *con /* --------------------------------------------Timestamp Compression * ---------------------------------------------- */ // TODO: Take care here, we assumes little endian encoding. -int tsCompressTimestampImp(const char *const input, const int nelements, char *const output) { - int _pos = 1; +int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { + int32_t _pos = 1; assert(nelements >= 0); if (nelements == 0) return 0; int64_t *istream = (int64_t *)input; - int64_t prev_value = istream[0]; - if(prev_value >= 0x8000000000000000) { - uWarn("compression timestamp is over signed long long range. ts = 0x%"PRIx64" \n", prev_value); - goto _exit_over; + int64_t prev_value = istream[0]; + if (prev_value >= 0x8000000000000000) { + uWarn("compression timestamp is over signed long long range. ts = 0x%" PRIx64 " \n", prev_value); + goto _exit_over; } int64_t prev_delta = -prev_value; uint8_t flags = 0, flag1 = 0, flag2 = 0; uint64_t dd1 = 0, dd2 = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { int64_t curr_value = istream[i]; if (!safeInt64Add(curr_value, -prev_value)) goto _exit_over; int64_t curr_delta = curr_value - prev_value; @@ -564,7 +561,7 @@ _exit_over: return nelements * LONG_BYTES + 1; } -int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { assert(nelements >= 0); if (nelements == 0) return 0; @@ -574,7 +571,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char } else if (input[0] == 1) { // Decompress int64_t *ostream = (int64_t *)output; - int ipos = 1, opos = 0; + int32_t ipos = 1, opos = 0; int8_t nbytes = 0; int64_t prev_value = 0; int64_t prev_delta = 0; @@ -635,9 +632,9 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char } /* --------------------------------------------Double Compression * ---------------------------------------------- */ -void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int *const pos) { +void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int32_t *const pos) { uint8_t nbytes = (flag & INT8MASK(3)) + 1; - int nshift = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t nshift = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff >>= nshift; while (nbytes) { @@ -647,9 +644,9 @@ void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int *con } } -int tsCompressDoubleImp(const char *const input, const int nelements, char *const output) { - int byte_limit = nelements * DOUBLE_BYTES + 1; - int opos = 1; +int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output) { + int32_t byte_limit = nelements * DOUBLE_BYTES + 1; + int32_t opos = 1; uint64_t prev_value = 0; uint64_t prev_diff = 0; @@ -658,7 +655,7 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons double *istream = (double *)input; // Main loop - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { union { double real; uint64_t bits; @@ -670,8 +667,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons uint64_t predicted = prev_value; uint64_t diff = curr.bits ^ predicted; - int leading_zeros = LONG_BYTES * BITS_PER_BYTE; - int trailing_zeros = leading_zeros; + int32_t leading_zeros = LONG_BYTES * BITS_PER_BYTE; + int32_t trailing_zeros = leading_zeros; if (diff) { trailing_zeros = BUILDIN_CTZL(diff); @@ -696,8 +693,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons prev_diff = diff; prev_flag = flag; } else { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = (flag & INT8MASK(3)) + 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = (flag & INT8MASK(3)) + 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag | (flag << 4); output[opos++] = flags; @@ -713,8 +710,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons } if (nelements % 2) { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag; output[opos++] = flags; @@ -731,19 +728,19 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons return opos; } -uint64_t decodeDoubleValue(const char *const input, int *const ipos, uint8_t flag) { +uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint64_t diff = 0ul; - int nbytes = (flag & INT8MASK(3)) + 1; - for (int i = 0; i < nbytes; i++) { + int32_t nbytes = (flag & INT8MASK(3)) + 1; + for (int32_t i = 0; i < nbytes; i++) { diff = diff | ((INT64MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); } - int shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; return diff; } -int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output) { // output stream double *ostream = (double *)output; @@ -753,11 +750,11 @@ int tsDecompressDoubleImp(const char *const input, const int nelements, char *co } uint8_t flags = 0; - int ipos = 1; - int opos = 0; + int32_t ipos = 1; + int32_t opos = 0; uint64_t prev_value = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % 2 == 0) { flags = input[ipos++]; } @@ -783,9 +780,9 @@ int tsDecompressDoubleImp(const char *const input, const int nelements, char *co /* --------------------------------------------Float Compression * ---------------------------------------------- */ -void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int *const pos) { +void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int32_t *const pos) { uint8_t nbytes = (flag & INT8MASK(3)) + 1; - int nshift = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t nshift = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff >>= nshift; while (nbytes) { @@ -795,17 +792,17 @@ void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int *cons } } -int tsCompressFloatImp(const char *const input, const int nelements, char *const output) { - float *istream = (float *)input; - int byte_limit = nelements * FLOAT_BYTES + 1; - int opos = 1; +int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output) { + float *istream = (float *)input; + int32_t byte_limit = nelements * FLOAT_BYTES + 1; + int32_t opos = 1; uint32_t prev_value = 0; uint32_t prev_diff = 0; uint8_t prev_flag = 0; // Main loop - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { union { float real; uint32_t bits; @@ -817,8 +814,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const uint32_t predicted = prev_value; uint32_t diff = curr.bits ^ predicted; - int leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; - int trailing_zeros = leading_zeros; + int32_t leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; + int32_t trailing_zeros = leading_zeros; if (diff) { trailing_zeros = BUILDIN_CTZ(diff); @@ -843,8 +840,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const prev_diff = diff; prev_flag = flag; } else { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = (flag & INT8MASK(3)) + 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = (flag & INT8MASK(3)) + 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag | (flag << 4); output[opos++] = flags; @@ -860,8 +857,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const } if (nelements % 2) { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag; output[opos++] = flags; @@ -878,19 +875,19 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const return opos; } -uint32_t decodeFloatValue(const char *const input, int *const ipos, uint8_t flag) { +uint32_t decodeFloatValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint32_t diff = 0ul; - int nbytes = (flag & INT8MASK(3)) + 1; - for (int i = 0; i < nbytes; i++) { + int32_t nbytes = (flag & INT8MASK(3)) + 1; + for (int32_t i = 0; i < nbytes; i++) { diff = diff | ((INT32MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); } - int shift_width = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t shift_width = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; return diff; } -int tsDecompressFloatImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output) { float *ostream = (float *)output; if (input[0] == 1) { @@ -899,11 +896,11 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con } uint8_t flags = 0; - int ipos = 1; - int opos = 0; + int32_t ipos = 1; + int32_t opos = 0; uint32_t prev_value = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % 2 == 0) { flags = input[ipos++]; } @@ -927,15 +924,15 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con return nelements * FLOAT_BYTES; } -#ifdef TD_TSZ +#ifdef TD_TSZ // // ---------- float double lossy ----------- // -int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output){ - // compress with sz - int compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1); +int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output) { + // compress with sz + int32_t compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1); unsigned char algo = ALGO_SZ_LOSSY << 1; - if (compressedSize == 0 || compressedSize >= nelements*sizeof(float)){ + if (compressedSize == 0 || compressedSize >= nelements * sizeof(float)) { // compressed error or large than original output[0] = MODE_NOCOMPRESS | algo; memcpy(output + 1, input, nelements * sizeof(float)); @@ -949,25 +946,26 @@ int tsCompressFloatLossyImp(const char * input, const int nelements, char *const return compressedSize; } -int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ - int decompressedSize = 0; - if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ +int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output) { + int32_t decompressedSize = 0; + if (HEAD_MODE(input[0]) == MODE_NOCOMPRESS) { // orginal so memcpy directly decompressedSize = nelements * sizeof(float); memcpy(output, input + 1, decompressedSize); return decompressedSize; - } + } // decompressed with sz return tdszDecompress(SZ_FLOAT, input + 1, compressedSize - 1, nelements, output); } -int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output){ - // compress with sz - int compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1); +int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output) { + // compress with sz + int32_t compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1); unsigned char algo = ALGO_SZ_LOSSY << 1; - if (compressedSize == 0 || compressedSize >= nelements*sizeof(double)) { + if (compressedSize == 0 || compressedSize >= nelements * sizeof(double)) { // compressed error or large than original output[0] = MODE_NOCOMPRESS | algo; memcpy(output + 1, input, nelements * sizeof(double)); @@ -978,18 +976,19 @@ int tsCompressDoubleLossyImp(const char * input, const int nelements, char *cons compressedSize += 1; } - return compressedSize; + return compressedSize; } -int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ - int decompressedSize = 0; - if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ +int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output) { + int32_t decompressedSize = 0; + if (HEAD_MODE(input[0]) == MODE_NOCOMPRESS) { // orginal so memcpy directly decompressedSize = nelements * sizeof(double); memcpy(output, input + 1, decompressedSize); return decompressedSize; - } + } // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index dc9fa33595defda2234d51f4c936a02551b7cefb..8330c10ec703b7c2484167ff5cba7d46b9eabe85 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -16,19 +16,12 @@ #define _DEFAULT_SOURCE #include "tconfig.h" #include "taoserror.h" -#include "tcfg.h" -#include "thash.h" -#include "tutil.h" #include "tlog.h" +#include "tutil.h" #define CFG_NAME_PRINT_LEN 24 #define CFG_SRC_PRINT_LEN 12 -typedef struct SConfig { - ECfgSrcType stype; - SHashObj *hash; -} SConfig; - int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromEnvVar(SConfig *pConfig); @@ -42,8 +35,8 @@ SConfig *cfgInit() { return NULL; } - pCfg->hash = taosHashInit(16, MurmurHash3_32, false, HASH_NO_LOCK); - if (pCfg->hash == NULL) { + pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); + if (pCfg->array == NULL) { free(pCfg); terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -92,25 +85,18 @@ static void cfgFreeItem(SConfigItem *pItem) { void cfgCleanup(SConfig *pCfg) { if (pCfg != NULL) { - if (pCfg->hash != NULL) { - SConfigItem *pItem = taosHashIterate(pCfg->hash, NULL); - while (pItem != NULL) { - cfgFreeItem(pItem); - tfree(pItem->name); - pItem = taosHashIterate(pCfg->hash, pItem); - } - taosHashCleanup(pCfg->hash); - pCfg->hash == NULL; + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + cfgFreeItem(pItem); + tfree(pItem->name); } + taosArrayDestroy(pCfg->array); free(pCfg); } } -int32_t cfgGetSize(SConfig *pCfg) { return taosHashGetSize(pCfg->hash); } - -SConfigItem *cfgIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashIterate(pCfg->hash, pIter); } - -void cfgCancelIterate(SConfig *pCfg, SConfigItem *pIter) { return taosHashCancelIterate(pCfg->hash, pIter); } +int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); } static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { cfgFreeItem(pItem); @@ -359,16 +345,16 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy } SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { - int32_t len = strlen(name); - char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; - strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - - SConfigItem *pItem = taosHashGet(pCfg->hash, lowcaseName, len + 1); - if (pItem == NULL) { - terrno = TSDB_CODE_CFG_NOT_FOUND; + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + if (strcasecmp(pItem->name, name) == 0) { + return pItem; + } } - return pItem; + terrno = TSDB_CODE_CFG_NOT_FOUND; + return NULL; } static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { @@ -383,7 +369,7 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - if (taosHashPut(pCfg->hash, lowcaseName, len + 1, pItem, sizeof(SConfigItem)) != 0) { + if (taosArrayPush(pCfg->array, pItem) == NULL) { if (pItem->dtype == CFG_DTYPE_STRING) { free(pItem->str); } @@ -536,8 +522,9 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { char src[CFG_SRC_PRINT_LEN + 1] = {0}; char name[CFG_NAME_PRINT_LEN + 1] = {0}; - SConfigItem *pItem = cfgIterate(pCfg, NULL); - while (pItem != NULL) { + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && !pItem->tsc) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { @@ -596,7 +583,6 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { } break; } - pItem = cfgIterate(pCfg, pItem); } if (dump) { @@ -619,11 +605,11 @@ int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { char *line = NULL, *name, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; + int32_t olen, vlen, vlen2, vlen3; ssize_t _bytes = 0; // FILE *fp = fopen(filepath, "r"); - TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index 73bba4480e99f45a41ce937e9ea99bcd9077e995..f7e1060be0c42fa3704c9a471e5ae5719bdf90e2 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tcrc32c.h" #include "tdef.h" @@ -26,8 +26,6 @@ #include #endif - - #define POLY 0x82f63b78 #define LONG_SHIFT 8192 #define SHORT_SHIFT 256 @@ -1097,7 +1095,7 @@ static uint32_t short_shifts[4][256] = { static uint32_t append_trivial(uint32_t crc, crc_stream input, size_t length) { for (size_t i = 0; i < length; ++i) { crc = crc ^ input[i]; - for (int j = 0; j < 8; j++) + for (int32_t j = 0; j < 8; j++) crc = (crc >> 1) ^ 0x80000000 ^ ((~crc & 1) * POLY); } return crc; @@ -1358,7 +1356,7 @@ void taosResolveCRC() { #if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS crc32c = crc32c_sf; #else - int sse42; + int32_t sse42; SSE42(sse42); crc32c = sse42 ? crc32c_hw : crc32c_sf; #endif @@ -1371,10 +1369,10 @@ void taosResolveCRC() { #include #include -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char str[1024] = "\0"; char *ptr = str; - int count = 0; + int32_t count = 0; while ((count = read(0, ptr, 10)) > 0) { ptr += count; } diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index f72ddcaa3bb1c63e06310f574dcaa765ed112046..105dd7f95fc204d11ad9233372984944760dfd70 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -13,43 +13,44 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tkey.h" +#define _DEFAULT_SOURCE +#include "tdes.h" #define ENCRYPTION_MODE 1 #define DECRYPTION_MODE 0 typedef struct { - unsigned char k[8]; - unsigned char c[4]; - unsigned char d[4]; + uint8_t k[8]; + uint8_t c[4]; + uint8_t d[4]; } key_set; -void generate_key(unsigned char* key); -void generate_sub_keys(unsigned char* main_key, key_set* key_sets); -void process_message(unsigned char* message_piece, unsigned char* processed_piece, key_set* key_sets, int mode); +void generate_key(uint8_t* key); +void generate_sub_keys(uint8_t* main_key, key_set* key_sets); +void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* key_sets, int32_t mode); #if 0 int64_t taosDesGenKey() { - unsigned int iseed = (unsigned int)time(NULL); + uint32_t iseed = (uint32_t)time(NULL); srand(iseed); - unsigned char key[8] = {0}; + uint8_t key[8] = {0}; generate_key(key); return *((int64_t*)key); } #endif -char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mode) { - unsigned int number_of_blocks = len / 8; - unsigned char data_block[9] = {0}; - unsigned char processed_block[9] = {0}; - key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); - char* dest = calloc(len + 1, 1); +char* taosDesImp(uint8_t* key, char* src, uint32_t len, int32_t process_mode) { + uint32_t number_of_blocks = len / 8; + uint8_t data_block[9] = {0}; + uint8_t processed_block[9] = {0}; + key_set key_sets[17]; + memset(key_sets, 0, sizeof(key_sets)); + char* dest = calloc(len + 1, 1); generate_sub_keys(key, key_sets); - for (unsigned int block_count = 0; block_count < number_of_blocks; block_count++) { + for (uint32_t block_count = 0; block_count < number_of_blocks; block_count++) { memset(processed_block, 0, 8); memcpy(data_block, src + block_count * 8, 8); process_message(data_block, processed_block, key_sets, process_mode); @@ -59,15 +60,15 @@ char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mo return dest; } -char* taosDesEncode(int64_t key, char* src, int len) { +char* taosDesEncode(int64_t key, char* src, int32_t len) { if (len % 8 != 0) return NULL; - unsigned char* keyStr = (unsigned char*)(&key); + uint8_t* keyStr = (uint8_t*)(&key); return taosDesImp(keyStr, src, len, ENCRYPTION_MODE); } -char* taosDesDecode(int64_t key, char* src, int len) { - unsigned char* keyStr = (unsigned char*)(&key); - char* temp = calloc(len + 8, 1); +char* taosDesDecode(int64_t key, char* src, int32_t len) { + uint8_t* keyStr = (uint8_t*)(&key); + char* temp = calloc(len + 8, 1); memcpy(temp, src, len); len += 8; @@ -77,67 +78,67 @@ char* taosDesDecode(int64_t key, char* src, int len) { return decode; } -int initial_key_permutaion[] = {57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, - 35, 27, 19, 11, 3, 60, 52, 44, 36, 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, - 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4}; +int32_t initial_key_permutaion[] = {57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, + 35, 27, 19, 11, 3, 60, 52, 44, 36, 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, + 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4}; -int initial_message_permutation[] = {58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, - 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, - 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, - 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7}; +int32_t initial_message_permutation[] = {58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, + 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, + 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, + 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7}; -int key_shift_sizes[] = {-1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1}; +int32_t key_shift_sizes[] = {-1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1}; -int sub_key_permutation[] = {14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, - 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47, 55, 30, 40, - 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32}; +int32_t sub_key_permutation[] = {14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, + 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47, 55, 30, 40, + 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32}; -int message_expansion[] = {32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, - 12, 13, 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, - 22, 23, 24, 25, 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1}; +int32_t message_expansion[] = {32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, + 12, 13, 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, + 22, 23, 24, 25, 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1}; -int S1[] = {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, 0, 15, 7, 4, 14, 2, - 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, - 3, 10, 5, 0, 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}; +int32_t S1[] = {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, 0, 15, 7, 4, 14, 2, + 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, + 3, 10, 5, 0, 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}; -int S2[] = {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, 3, 13, 4, 7, 15, 2, - 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, - 9, 3, 2, 15, 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}; +int32_t S2[] = {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, 3, 13, 4, 7, 15, 2, + 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, + 9, 3, 2, 15, 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}; -int S3[] = {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, 13, 7, 0, 9, 3, 4, - 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, - 5, 10, 14, 7, 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}; +int32_t S3[] = {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, 13, 7, 0, 9, 3, 4, + 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, + 5, 10, 14, 7, 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}; -int S4[] = {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, 13, 8, 11, 5, 6, 15, - 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, - 5, 2, 8, 4, 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}; +int32_t S4[] = {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, 13, 8, 11, 5, 6, 15, + 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, + 5, 2, 8, 4, 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}; -int S5[] = {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, 14, 11, 2, 12, 4, 7, - 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, - 6, 3, 0, 14, 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}; +int32_t S5[] = {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, 14, 11, 2, 12, 4, 7, + 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, + 6, 3, 0, 14, 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}; -int S6[] = {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, 10, 15, 4, 2, 7, 12, - 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, - 1, 13, 11, 6, 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}; +int32_t S6[] = {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, 10, 15, 4, 2, 7, 12, + 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, + 1, 13, 11, 6, 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}; -int S7[] = {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, 13, 0, 11, 7, 4, 9, - 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, - 0, 5, 9, 2, 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}; +int32_t S7[] = {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, 13, 0, 11, 7, 4, 9, + 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, + 0, 5, 9, 2, 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}; -int S8[] = {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, 1, 15, 13, 8, 10, 3, - 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, - 15, 3, 5, 8, 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}; +int32_t S8[] = {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, 1, 15, 13, 8, 10, 3, + 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, + 15, 3, 5, 8, 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}; -int right_sub_message_permutation[] = {16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, - 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25}; +int32_t right_sub_message_permutation[] = {16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, + 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25}; -int final_message_permutation[] = {40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, - 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, - 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, - 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25}; +int32_t final_message_permutation[] = {40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, + 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, + 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, + 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25}; void print_char_as_binary(char input) { - int i; + int32_t i; for (i = 0; i < 8; i++) { char shift_byte = 0x01 << (7 - i); if (shift_byte & input) { @@ -148,15 +149,15 @@ void print_char_as_binary(char input) { } } -void generate_key(unsigned char* key) { - int i; +void generate_key(uint8_t* key) { + int32_t i; for (i = 0; i < 8; i++) { key[i] = taosRand() % 255; } } void print_key_set(key_set _key_set) { - int i; + int32_t i; printf("K: \n"); for (i = 0; i < 8; i++) { printf("%02X : ", _key_set.k[i]); @@ -180,10 +181,10 @@ void print_key_set(key_set _key_set) { printf("\n"); } -void generate_sub_keys(unsigned char* main_key, key_set* key_sets) { - int i, j; - int shift_size; - unsigned char shift_byte, first_shift_bits, second_shift_bits, third_shift_bits, fourth_shift_bits; +void generate_sub_keys(uint8_t* main_key, key_set* key_sets) { + int32_t i, j; + int32_t shift_size; + uint8_t shift_byte, first_shift_bits, second_shift_bits, third_shift_bits, fourth_shift_bits; for (i = 0; i < 8; i++) { key_sets[0].k[i] = 0; @@ -277,12 +278,12 @@ void generate_sub_keys(unsigned char* main_key, key_set* key_sets) { } } -void process_message(unsigned char* message_piece, unsigned char* processed_piece, key_set* key_sets, int mode) { - int i, k; - int shift_size; - unsigned char shift_byte; +void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* key_sets, int32_t mode) { + int32_t i, k; + int32_t shift_size; + uint8_t shift_byte; - unsigned char initial_permutation[8]; + uint8_t initial_permutation[8]; memset(initial_permutation, 0, 8); memset(processed_piece, 0, 8); @@ -295,15 +296,15 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec initial_permutation[i / 8] |= (shift_byte >> i % 8); } - unsigned char l[4], r[4]; + uint8_t l[4], r[4]; for (i = 0; i < 4; i++) { l[i] = initial_permutation[i]; r[i] = initial_permutation[i + 4]; } - unsigned char ln[4], rn[4], er[6], ser[4]; + uint8_t ln[4], rn[4], er[6], ser[4]; - int key_index; + int32_t key_index; for (k = 1; k <= 16; k++) { memcpy(ln, r, 4); @@ -328,7 +329,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec er[i] ^= key_sets[key_index].k[i]; } - unsigned char row, column; + uint8_t row, column; for (i = 0; i < 4; i++) { ser[i] = 0; @@ -345,7 +346,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[0] & 0x78) >> 3); - ser[0] |= ((unsigned char)S1[row * 16 + column] << 4); + ser[0] |= ((uint8_t)S1[row * 16 + column] << 4); row = 0; row |= (er[0] & 0x02); @@ -355,7 +356,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[0] & 0x01) << 3); column |= ((er[1] & 0xE0) >> 5); - ser[0] |= (unsigned char)S2[row * 16 + column]; + ser[0] |= (uint8_t)S2[row * 16 + column]; // Byte 2 row = 0; @@ -366,7 +367,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[1] & 0x07) << 1); column |= ((er[2] & 0x80) >> 7); - ser[1] |= ((unsigned char)S3[row * 16 + column] << 4); + ser[1] |= ((uint8_t)S3[row * 16 + column] << 4); row = 0; row |= ((er[2] & 0x20) >> 4); @@ -375,7 +376,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[2] & 0x1E) >> 1); - ser[1] |= (unsigned char)S4[row * 16 + column]; + ser[1] |= (uint8_t)S4[row * 16 + column]; // Byte 3 row = 0; @@ -385,7 +386,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[3] & 0x78) >> 3); - ser[2] |= ((unsigned char)S5[row * 16 + column] << 4); + ser[2] |= ((uint8_t)S5[row * 16 + column] << 4); row = 0; row |= (er[3] & 0x02); @@ -395,7 +396,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[3] & 0x01) << 3); column |= ((er[4] & 0xE0) >> 5); - ser[2] |= (unsigned char)S6[row * 16 + column]; + ser[2] |= (uint8_t)S6[row * 16 + column]; // Byte 4 row = 0; @@ -406,7 +407,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[4] & 0x07) << 1); column |= ((er[5] & 0x80) >> 7); - ser[3] |= ((unsigned char)S7[row * 16 + column] << 4); + ser[3] |= ((uint8_t)S7[row * 16 + column] << 4); row = 0; row |= ((er[5] & 0x20) >> 4); @@ -415,7 +416,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[5] & 0x1E) >> 1); - ser[3] |= (unsigned char)S8[row * 16 + column]; + ser[3] |= (uint8_t)S8[row * 16 + column]; for (i = 0; i < 4; i++) { rn[i] = 0; @@ -440,7 +441,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec } } - unsigned char pre_end_permutation[8]; + uint8_t pre_end_permutation[8]; for (i = 0; i < 4; i++) { pre_end_permutation[i] = r[i]; pre_end_permutation[4 + i] = l[i]; diff --git a/source/util/src/encode.c b/source/util/src/tencode.c similarity index 96% rename from source/util/src/encode.c rename to source/util/src/tencode.c index 758d3f442db0007ea60317827d582d50f60f8d75..94b4cced46adb7f72e89b27b415a52ccf48a0253 100644 --- a/source/util/src/encode.c +++ b/source/util/src/tencode.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "encode.h" +#define _DEFAULT_SOURCE +#include "tencode.h" #if __STDC_VERSION__ >= 201112L static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); @@ -47,7 +48,7 @@ void tCoderClear(SCoder* pCoder) { } } -int tStartEncode(SCoder* pCoder) { +int32_t tStartEncode(SCoder* pCoder) { struct SCoderNode* pNode; ASSERT(pCoder->type == TD_ENCODER); @@ -96,7 +97,7 @@ void tEndEncode(SCoder* pCoder) { } } -int tStartDecode(SCoder* pCoder) { +int32_t tStartDecode(SCoder* pCoder) { int32_t len; struct SCoderNode* pNode; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index cdc8515148b98126831831667fbc4b9746c563ad..c8cafb93dde0b08edc329ebba1394bb047c3518b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -243,6 +243,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +// mnode-infoSchema +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name") + + // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") diff --git a/source/util/src/texception.c b/source/util/src/texception.c new file mode 100644 index 0000000000000000000000000000000000000000..33befb694a0a0a71c01042d9c9b5b19f7da12912 --- /dev/null +++ b/source/util/src/texception.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "texception.h" + +static threadlocal SExceptionNode* expList; + +void exceptionPushNode(SExceptionNode* node) { + node->prev = expList; + expList = node; +} + +int32_t exceptionPopNode() { + SExceptionNode* node = expList; + expList = node->prev; + return node->code; +} + +void exceptionThrow(int32_t code) { + expList->code = code; + longjmp(expList->jb, 1); +} + +static void cleanupWrapper_void_ptr_ptr(SCleanupAction* ca) { + void (*func)(void*, void*) = ca->func; + func(ca->arg1.Ptr, ca->arg2.Ptr); +} + +static void cleanupWrapper_void_ptr_bool(SCleanupAction* ca) { + void (*func)(void*, bool) = ca->func; + func(ca->arg1.Ptr, ca->arg2.Bool); +} + +static void cleanupWrapper_void_ptr(SCleanupAction* ca) { + void (*func)(void*) = ca->func; + func(ca->arg1.Ptr); +} + +static void cleanupWrapper_int_int(SCleanupAction* ca) { + int32_t (*func)(int32_t) = ca->func; + func(ca->arg1.Int); +} + +static void cleanupWrapper_void(SCleanupAction* ca) { + void (*func)() = ca->func; + func(); +} + +static void cleanupWrapper_int_ptr(SCleanupAction* ca) { + int32_t (*func)(void*) = ca->func; + func(ca->arg1.Ptr); +} + +typedef void (*wrapper)(SCleanupAction*); +static wrapper wrappers[] = { + cleanupWrapper_void_ptr_ptr, cleanupWrapper_void_ptr_bool, cleanupWrapper_void_ptr, + cleanupWrapper_int_int, cleanupWrapper_void, cleanupWrapper_int_ptr, +}; + +void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 0; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Ptr = arg2; +} + +void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 1; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Bool = arg2; +} + +void cleanupPush_void_ptr(bool failOnly, void* func, void* arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 2; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} + +void cleanupPush_int_int(bool failOnly, void* func, int32_t arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 3; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Int = arg; +} + +void cleanupPush_void(bool failOnly, void* func) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 4; + ca->failOnly = failOnly; + ca->func = func; +} + +void cleanupPush_int_ptr(bool failOnly, void* func, void* arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 5; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} + +int32_t cleanupGetActionCount() { return expList->numCleanupAction; } + +static void doExecuteCleanup(SExceptionNode* node, int32_t anchor, bool failed) { + while (node->numCleanupAction > anchor) { + --node->numCleanupAction; + SCleanupAction* ca = node->cleanupActions + node->numCleanupAction; + if (failed || !(ca->failOnly)) { + wrappers[ca->wrapper](ca); + } + } +} + +void cleanupExecuteTo(int32_t anchor, bool failed) { doExecuteCleanup(expList, anchor, failed); } + +void cleanupExecute(SExceptionNode* node, bool failed) { doExecuteCleanup(node, 0, failed); } +bool cleanupExceedLimit() { return expList->numCleanupAction >= expList->maxCleanupAction; } diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c deleted file mode 100644 index ef9708df0ccf61852e1e5f465a9670a72916efac..0000000000000000000000000000000000000000 --- a/source/util/src/tfile.c +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tref.h" -#include "tutil.h" -#include "tlog.h" - -static int32_t tsFileRsetId = -1; - -static int8_t tfInited = 0; - -// static void tfCloseFile(void *p) { taosCloseFile((TdFilePtr)(uintptr_t)p); } - -int32_t tfInit() { - // int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - // if (old == 1) return 0; - // tsFileRsetId = taosOpenRef(2000, tfCloseFile); - // if (tsFileRsetId > 0) { - // return 0; - // } else { - // atomic_store_8(&tfInited, 0); - // return -1; - // } -} - -void tfCleanup() { - // atomic_store_8(&tfInited, 0); - // if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - // tsFileRsetId = -1; -} - -// static int64_t tfOpenImp(TdFilePtr pFile) { -// if (pFile == NULL) { -// terrno = TAOS_SYSTEM_ERROR(errno); -// return -1; -// } - -// void * p = (void *)(int64_t)pFile; -// int64_t rid = taosAddRef(tsFileRsetId, p); -// if (rid < 0) taosCloseFile(&pFile); - -// return rid; -// } - -// int64_t tfOpenRead(const char *pathname, int32_t flags) { -// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ); -// return tfOpenImp(fd); -// } - -// int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { -// int32_t pFile = taosOpenFile(pathname, TD_FILE_READ | TD_FILE_WRITE); -// return tfOpenImp(fd); -// } - -// int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { -// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE); -// return tfOpenImp(fd); -// } - -// int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { -// int32_t pFile = taosOpenFile(pathname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); -// return tfOpenImp(fd); -// } - -// int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } - -// int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; - -// int64_t ret = taosWriteFile(pFile, buf, count); -// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - -// taosReleaseRef(tsFileRsetId, tfd); -// return ret; -// } - -// int64_t tfRead(int64_t tfd, void *buf, int64_t count) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; - -// int64_t ret = taosReadFile(pFile, buf, count); -// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - -// taosReleaseRef(tsFileRsetId, tfd); -// return ret; -// } - -// int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; - -// int64_t ret = pread(fd, buf, count, offset); -// if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - -// taosReleaseRef(tsFileRsetId, tfd); -// return ret; -// } - -// int32_t tfFsync(int64_t tfd) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; -// int32_t code = taosFsyncFile(pFile); - -// taosReleaseRef(tsFileRsetId, tfd); -// return code; -// } - -// bool tfValid(int64_t tfd) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return false; - -// taosReleaseRef(tsFileRsetId, tfd); -// return true; -// } - -// int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; -// int64_t ret = taosLSeekFile(fd, offset, whence); - -// taosReleaseRef(tsFileRsetId, tfd); -// return ret; -// } - -// int32_t tfFtruncate(int64_t tfd, int64_t length) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return -1; - -// int32_t pFile = (TdFilePtr)(uintptr_t)p; -// int32_t code = taosFtruncateFile(fd, length); - -// taosReleaseRef(tsFileRsetId, tfd); -// return code; -// } - -// void *tfMmapReadOnly(int64_t tfd, int64_t length) { -// void *p = taosAcquireRef(tsFileRsetId, tfd); -// if (p == NULL) return NULL; -// int32_t pFile = (TdFilePtr)(uintptr_t)p; - -// void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); -// taosReleaseRef(tsFileRsetId, tfd); -// return ptr; -// } diff --git a/source/util/src/tfunctional.c b/source/util/src/tfunctional.c index 8b20f8fc0a6b6e08bcfd501f625c4594b4dff50d..c49fbdb50438effa0d980d50aaa00ea4a2d9428b 100644 --- a/source/util/src/tfunctional.c +++ b/source/util/src/tfunctional.c @@ -13,37 +13,34 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tfunctional.h" -tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs) { +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) { tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); - if(pSavedFunc == NULL) return NULL; + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs) { - tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void *)); - if(pSavedFunc == NULL) return NULL; +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) { + tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs) { +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) { tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); - if(pSavedFunc == NULL) return NULL; + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { - return pSavedFunc->func(pSavedFunc->args); -} +FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); } -FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { - return pSavedFunc->func(pSavedFunc->args); -} +FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); } FORCE_INLINE void voidInvoke(tVoidSavedFunc* const pSavedFunc) { - if(pSavedFunc) pSavedFunc->func(pSavedFunc->args); + if (pSavedFunc) pSavedFunc->func(pSavedFunc->args); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 9f39b79968c2684681ee508915b8e640598cc2f8..219fc739ca845e1abf5b6a55e1ce1c0a87cb6149 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -13,16 +13,15 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "thash.h" -#include "tlog.h" -#include "taos.h" #include "tdef.h" +#include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT #define MAX_WARNING_REF_COUNT 10000 #define EXT_SIZE 1024 -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define DO_FREE_HASH_NODE(_n) \ do { \ @@ -74,10 +73,12 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, + uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -115,7 +116,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, + SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); pNode->count--; @@ -129,11 +131,11 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pNewNode->next = pNode->next; DO_FREE_HASH_NODE(pNode); } else { - pNewNode->next = pNode; + pNewNode->next = pNode; pe->num++; atomic_add_fetch_32(&pHashObj->size, 1); } - + return pNewNode; } @@ -175,7 +177,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->capacity = taosHashCapacity((int32_t)capacity); assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; @@ -201,8 +203,8 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { if (pHashObj != NULL && fp != NULL) { pHashObj->equalFp = fp; - } -} + } +} int32_t taosHashGetSize(const SHashObj *pHashObj) { if (!pHashObj) { @@ -211,9 +213,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)atomic_load_32(&pHashObj->size); } -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -224,12 +224,12 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock((void*) &pHashObj->lock, pHashObj->type); + __wr_lock((void *)&pHashObj->lock, pHashObj->type); taosHashTableResize(pHashObj); - __wr_unlock((void*) &pHashObj->lock, pHashObj->type); + __wr_unlock((void *)&pHashObj->lock, pHashObj->type); } - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -245,9 +245,10 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void assert(pNode == NULL); } - SHashNode* prev = NULL; + SHashNode *prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -271,13 +272,13 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void } // enable resize - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); atomic_add_fetch_32(&pHashObj->size, 1); if (newAdded) { *newAdded = true; } - + return 0; } else { // not support the update operation, return error @@ -292,7 +293,7 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void } // enable resize - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); if (newAdded) { *newAdded = false; @@ -310,13 +311,13 @@ int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded); } - void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetClone(pHashObj, key, keyLen, NULL); } -//TODO(yihaoDeng), merge with taosHashGetClone -void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { +// TODO(yihaoDeng), merge with taosHashGetClone +void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, + size_t *sz) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -324,14 +325,14 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return NULL; } @@ -353,19 +354,19 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo if (fp != NULL) { fp(GET_HASH_NODE_DATA(pNode)); } - + if (*d == NULL) { - *sz = pNode->dataLen + EXT_SIZE; - *d = calloc(1, *sz); - } else if (*sz < pNode->dataLen){ *sz = pNode->dataLen + EXT_SIZE; - *d = realloc(*d, *sz); + *d = calloc(1, *sz); + } else if (*sz < pNode->dataLen) { + *sz = pNode->dataLen + EXT_SIZE; + *d = realloc(*d, *sz); } memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); - // just make runtime happy + // just make runtime happy if ((*sz) - pNode->dataLen > 0) { memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); - } + } data = GET_HASH_NODE_DATA(pNode); } @@ -374,11 +375,11 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo taosRUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void* d, bool acquire) { +void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *d, bool acquire) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -386,14 +387,14 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return NULL; } @@ -431,20 +432,19 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v taosRUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* d) { +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *d) { return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false); } -void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true); } - -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, void *data, size_t dsize*/) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen /*, void *data, size_t dsize*/) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return -1; } @@ -452,7 +452,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -466,23 +466,23 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi assert(pe->next == NULL); taosWUnLockLatch(&pe->latch); - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return -1; } - int code = -1; + int32_t code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) break; prevNode = pNode; pNode = pNode->next; } - if (pNode) { code = 0; // it is found @@ -495,7 +495,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi pe->next = pNode->next; } -// if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); + // if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); pe->num--; atomic_sub_fetch_32(&pHashObj->size, 1); @@ -507,7 +507,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi taosWUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return code; } @@ -518,7 +518,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi } // disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t numOfEntries = (int32_t)pHashObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { @@ -533,7 +533,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // todo remove the first node SHashNode *pNode = NULL; - while((pNode = pEntry->next) != NULL) { + while ((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; atomic_sub_fetch_32(&pHashObj->size, 1); @@ -582,7 +582,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi } } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return 0; } @@ -593,7 +593,7 @@ void taosHashClear(SHashObj *pHashObj) { SHashNode *pNode, *pNext; - __wr_lock((void*) &pHashObj->lock, pHashObj->type); + __wr_lock((void *)&pHashObj->lock, pHashObj->type); for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; @@ -617,7 +617,7 @@ void taosHashClear(SHashObj *pHashObj) { } atomic_store_32(&pHashObj->size, 0); - __wr_unlock((void*) &pHashObj->lock, pHashObj->type); + __wr_unlock((void *)&pHashObj->lock, pHashObj->type); } void taosHashCleanup(SHashObj *pHashObj) { @@ -676,7 +676,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; @@ -685,7 +685,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newSize - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); + void *p = calloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -756,15 +756,13 @@ void taosHashTableResize(SHashObj *pHashObj) { } else { assert(pe->next != NULL); } - } - } int64_t et = taosGetTimestampUs(); uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, - ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -775,12 +773,12 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; - pNewNode->dataLen = (uint32_t) dsize; - pNewNode->count = 1; + pNewNode->dataLen = (uint32_t)dsize; + pNewNode->count = 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -802,15 +800,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + + sizeof(SHashObj); } -FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { +FORCE_INLINE int32_t taosHashGetKey(void *data, void **key, size_t *keyLen) { if (NULL == data || NULL == key) { return -1; } - - SHashNode * node = GET_HASH_PNODE(data); + + SHashNode *node = GET_HASH_PNODE(data); *key = GET_HASH_NODE_KEY(node); if (keyLen) { *keyLen = node->keyLen; @@ -820,19 +819,17 @@ FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { } FORCE_INLINE int32_t taosHashGetDataLen(void *data) { - SHashNode * node = GET_HASH_PNODE(data); + SHashNode *node = GET_HASH_PNODE(data); return node->keyLen; } FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { - SHashNode * node = GET_HASH_PNODE(data); + SHashNode *node = GET_HASH_PNODE(data); return node->keyLen; } - // release the pNode, return next pNode, and lock the current entry -static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { - +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); SHashNode *prevNode = NULL; @@ -847,14 +844,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pNode = pe->next; while (pNode) { - if (pNode == pOld) - break; + if (pNode == pOld) break; prevNode = pNode; pNode = pNode->next; } - if (pNode) { + if (pNode) { pNode = pNode->next; while (pNode) { if (pNode->removed == 0) break; @@ -862,17 +858,17 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } pOld->count--; - if (pOld->count <=0) { + if (pOld->count <= 0) { if (prevNode) { prevNode->next = pOld->next; } else { pe->next = pOld->next; } - + pe->num--; atomic_sub_fetch_32(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pOld); - } + } } else { uError("pNode:%p data:%p is not there!!!", pNode, p); } @@ -881,13 +877,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } void *taosHashIterate(SHashObj *pHashObj, void *p) { - if (pHashObj == NULL) return NULL; + if (pHashObj == NULL) return NULL; - int slot = 0; + int32_t slot = 0; char *data = NULL; // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); SHashNode *pNode = NULL; if (p) { @@ -896,7 +892,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { SHashEntry *pe = pHashObj->hashList[slot]; if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } slot = slot + 1; } @@ -921,7 +917,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } } } @@ -947,10 +943,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } @@ -958,21 +954,17 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL || p == NULL) return; // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); - int slot; + int32_t slot; taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } - - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); -} + } -void taosHashRelease(SHashObj *pHashObj, void *p) { - taosHashCancelIterate(pHashObj, p); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); } - +void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); } diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 2d7f8a5b032db7a2bfbc1d2cc6f6b541f76b1977..d5182cb892d2d314de52db7594768c00ad2808f1 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -13,10 +13,9 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "thash.h" -#include "compare.h" -#include "taos.h" +#include "tcompare.h" #include "types.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) @@ -34,7 +33,7 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint8_t *data = (const uint8_t *)key; - const int nblocks = len >> 2u; + const int32_t nblocks = len >> 2u; uint32_t h1 = 0x12345678; @@ -43,7 +42,7 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4); - for (int i = -nblocks; i; i++) { + for (int32_t i = -nblocks; i; i++) { uint32_t k1 = blocks[i]; k1 *= c1; @@ -92,7 +91,7 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { return 0; } if (fabs(f) < FLT_MAX/BASE - DLT) { - int t = (int)(round(BASE * (f + DLT))); + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; @@ -108,7 +107,7 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { return 0; } if (fabs(f) < DBL_MAX/BASE - DLT) { - int t = (int)(round(BASE * (f + DLT))); + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index aa822c7d5ec1d5f107385a53d77cc104c30b14a4..30af0483cc86c91b5ba2ba7a6fff81649f064a65 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "theap.h" -size_t heapSize(Heap* heap) { - return heap->nelts; -} +size_t heapSize(Heap* heap) { return heap->nelts; } Heap* heapCreate(HeapCompareFn fn) { - Heap *heap = calloc(1, sizeof(Heap)); - if (heap == NULL) { return NULL; } + Heap* heap = calloc(1, sizeof(Heap)); + if (heap == NULL) { + return NULL; + } heap->min = NULL; heap->nelts = 0; @@ -29,18 +30,14 @@ Heap* heapCreate(HeapCompareFn fn) { return heap; } -void heapDestroy(Heap *heap) { - free(heap); -} +void heapDestroy(Heap* heap) { free(heap); } -HeapNode* heapMin(const Heap* heap) { - return heap->min; -} +HeapNode* heapMin(const Heap* heap) { return heap->min; } /* Swap parent with child. Child moves closer to the root, parent moves away. */ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { HeapNode* sibling; - HeapNode t; + HeapNode t; t = *parent; *parent = *child; @@ -54,13 +51,10 @@ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { child->right = parent; sibling = child->left; } - if (sibling != NULL) - sibling->parent = child; + if (sibling != NULL) sibling->parent = child; - if (parent->left != NULL) - parent->left->parent = parent; - if (parent->right != NULL) - parent->right->parent = parent; + if (parent->left != NULL) parent->left->parent = parent; + if (parent->right != NULL) parent->right->parent = parent; if (child->parent == NULL) heap->min = child; @@ -73,9 +67,9 @@ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { void heapInsert(Heap* heap, HeapNode* newnode) { HeapNode** parent; HeapNode** child; - unsigned int path; - unsigned int n; - unsigned int k; + uint32_t path; + uint32_t n; + uint32_t k; newnode->left = NULL; newnode->right = NULL; @@ -85,8 +79,7 @@ void heapInsert(Heap* heap, HeapNode* newnode) { * heap so we always insert at the left-most free node of the bottom row. */ path = 0; - for (k = 0, n = 1 + heap->nelts; n >= 2; k += 1, n /= 2) - path = (path << 1) | (n & 1); + for (k = 0, n = 1 + heap->nelts; n >= 2; k += 1, n /= 2) path = (path << 1) | (n & 1); /* Now traverse the heap using the path we calculated in the previous step. */ parent = child = &heap->min; @@ -113,22 +106,20 @@ void heapInsert(Heap* heap, HeapNode* newnode) { } void heapRemove(Heap* heap, HeapNode* node) { - HeapNode* smallest; + HeapNode* smallest; HeapNode** max; - HeapNode* child; - unsigned int path; - unsigned int k; - unsigned int n; + HeapNode* child; + uint32_t path; + uint32_t k; + uint32_t n; - if (heap->nelts == 0) - return; + if (heap->nelts == 0) return; /* Calculate the path from the min (the root) to the max, the left-most node * of the bottom row. */ path = 0; - for (k = 0, n = heap->nelts; n >= 2; k += 1, n /= 2) - path = (path << 1) | (n & 1); + for (k = 0, n = heap->nelts; n >= 2; k += 1, n /= 2) path = (path << 1) | (n & 1); /* Now traverse the heap using the path we calculated in the previous step. */ max = &heap->min; @@ -182,12 +173,9 @@ void heapRemove(Heap* heap, HeapNode* node) { */ for (;;) { smallest = child; - if (child->left != NULL && (heap->compFn)(child->left, smallest)) - smallest = child->left; - if (child->right != NULL && (heap->compFn)(child->right, smallest)) - smallest = child->right; - if (smallest == child) - break; + if (child->left != NULL && (heap->compFn)(child->left, smallest)) smallest = child->left; + if (child->right != NULL && (heap->compFn)(child->right, smallest)) smallest = child->right; + if (smallest == child) break; heapNodeSwap(heap, child, smallest); } @@ -195,12 +183,7 @@ void heapRemove(Heap* heap, HeapNode* node) { * this is required, because `max` node is not guaranteed to be the * actual maximum in tree */ - while (child->parent != NULL && (heap->compFn)(child, child->parent)) - heapNodeSwap(heap, child->parent, child); -} - -void heapDequeue(Heap* heap) { - heapRemove(heap, heap->min); + while (child->parent != NULL && (heap->compFn)(child, child->parent)) heapNodeSwap(heap, child->parent, child); } - +void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); } diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index b4d76e6fb5bbd0c588e68a213d82e89b9d2424d5..3ae537eae8a9b7c5b16bc91af5c994c140179ee4 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -13,17 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "tidpool.h" #include "tlog.h" -typedef struct { - int maxId; - int numOfFree; - int freeSlot; - bool * freeList; - pthread_mutex_t mutex; -} id_pool_t; - -void *taosInitIdPool(int maxId) { +void *taosInitIdPool(int32_t maxId) { id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; @@ -44,17 +38,14 @@ void *taosInitIdPool(int maxId) { return pIdPool; } -int taosAllocateId(void *handle) { - id_pool_t *pIdPool = handle; - if (handle == NULL) { - return -1; - } +int32_t taosAllocateId(id_pool_t *pIdPool) { + if (pIdPool == NULL) return -1; - int slot = -1; + int32_t slot = -1; pthread_mutex_lock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { - for (int i = 0; i < pIdPool->maxId; ++i) { + for (int32_t i = 0; i < pIdPool->maxId; ++i) { slot = (i + pIdPool->freeSlot) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { pIdPool->freeList[slot] = true; @@ -69,13 +60,12 @@ int taosAllocateId(void *handle) { return slot + 1; } -void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool = handle; - if (handle == NULL) return; +void taosFreeId(id_pool_t *pIdPool, int32_t id) { + if (pIdPool == NULL) return; pthread_mutex_lock(&pIdPool->mutex); - int slot = (id - 1) % pIdPool->maxId; + int32_t slot = (id - 1) % pIdPool->maxId; if (pIdPool->freeList[slot]) { pIdPool->freeList[slot] = false; pIdPool->numOfFree++; @@ -84,9 +74,7 @@ void taosFreeId(void *handle, int id) { pthread_mutex_unlock(&pIdPool->mutex); } -void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool = handle; - +void taosIdPoolCleanUp(id_pool_t *pIdPool) { if (pIdPool == NULL) return; uDebug("pool:%p is cleaned", pIdPool); @@ -100,22 +88,19 @@ void taosIdPoolCleanUp(void *handle) { free(pIdPool); } -int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = handle; - +int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { pthread_mutex_lock(&pIdPool->mutex); - int ret = pIdPool->maxId - pIdPool->numOfFree; + int32_t ret = pIdPool->maxId - pIdPool->numOfFree; pthread_mutex_unlock(&pIdPool->mutex); return ret; } -bool taosIdPoolMarkStatus(void *handle, int id) { +bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { bool ret = false; - id_pool_t *pIdPool = handle; pthread_mutex_lock(&pIdPool->mutex); - int slot = (id - 1) % pIdPool->maxId; + int32_t slot = (id - 1) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { pIdPool->freeList[slot] = true; pIdPool->numOfFree--; @@ -128,8 +113,7 @@ bool taosIdPoolMarkStatus(void *handle, int id) { return ret; } -int taosUpdateIdPool(id_pool_t *handle, int maxId) { - id_pool_t *pIdPool = (id_pool_t*)handle; +int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { if (maxId <= pIdPool->maxId) { return 0; } @@ -154,11 +138,9 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return 0; } -int taosIdPoolMaxSize(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - +int32_t taosIdPoolMaxSize(id_pool_t *pIdPool) { pthread_mutex_lock(&pIdPool->mutex); - int ret = pIdPool->maxId; + int32_t ret = pIdPool->maxId; pthread_mutex_unlock(&pIdPool->mutex); return ret; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0c9d32ea13d8c216f40e247fbbf9e5e852ff9620..4b68467450a1e2f8b1313ca81953e295eb752483 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -13,22 +13,18 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tjson.h" - -#include "taoserror.h" #include "cJSON.h" +#include "taoserror.h" -SJson* tjsonCreateObject() { - return cJSON_CreateObject(); -} +SJson* tjsonCreateObject() { return cJSON_CreateObject(); } -void tjsonDelete(SJson* pJson) { - cJSON_Delete((cJSON*)pJson); -} +void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); } int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) { char tmp[40] = {0}; - snprintf(tmp, tListLen(tmp), "%"PRId64, number); + snprintf(tmp, tListLen(tmp), "%" PRId64, number); return tjsonAddStringToObject(pJson, pName, tmp); } @@ -44,11 +40,9 @@ int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { - return cJSON_AddArrayToObject((cJSON*)pJson, pName); -} +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); } -int32_t tjsonAddItemToObject(SJson *pJson, const char* pName, SJson* pItem) { +int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) { return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); } @@ -79,18 +73,11 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { return tjsonAddItemToArray(pJson, pJobj); } -char* tjsonToString(const SJson* pJson) { - return cJSON_Print((cJSON*)pJson); -} - -char* tjsonToUnformattedString(const SJson* pJson) { - return cJSON_PrintUnformatted((cJSON*)pJson); -} +char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } +char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } -SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { - return cJSON_GetObjectItem(pJson, pName); -} +SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); } int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); @@ -153,7 +140,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { uint64_t val = 0; - int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); *pVal = val; return code; } @@ -176,13 +163,9 @@ int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal) return TSDB_CODE_SUCCESS; } -int32_t tjsonGetArraySize(const SJson* pJson) { - return cJSON_GetArraySize(pJson); -} +int32_t tjsonGetArraySize(const SJson* pJson) { return cJSON_GetArraySize(pJson); } -SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index) { - return cJSON_GetArrayItem(pJson, index); -} +SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index) { return cJSON_GetArrayItem(pJson, index); } int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj) { SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); @@ -192,6 +175,4 @@ int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, voi return func(pJsonObj, pObj); } -SJson* tjsonParse(const char* pStr) { - return cJSON_Parse(pStr); -} +SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index f79bca1e4b03377cdfdaeda9ad4adf06ddff6de4..5fccba614b5c65beb6893ad0be64a61732bb200d 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -13,15 +13,15 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tlist.h" -#include "os.h" -void tdListInit(SList *list, int eleSize) { +void tdListInit(SList *list, int32_t eleSize) { TD_DLIST_INIT(list); listEleSize(list) = eleSize; } -SList *tdListNew(int eleSize) { +SList *tdListNew(int32_t eleSize) { SList *list = (SList *)malloc(sizeof(SList)); if (list == NULL) return NULL; @@ -50,7 +50,7 @@ void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, no void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } -int tdListPrepend(SList *list, void *data) { +int32_t tdListPrepend(SList *list, void *data) { SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; @@ -60,7 +60,7 @@ int tdListPrepend(SList *list, void *data) { return 0; } -int tdListAppend(SList *list, void *data) { +int32_t tdListAppend(SList *list, void *data) { SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; diff --git a/source/util/src/tlockfree.c b/source/util/src/tlockfree.c index f54206f5cbeee046fb009f9f3068b6305cd535e9..8edcca0c1635498e4fc25fa54028c26435f664f9 100644 --- a/source/util/src/tlockfree.c +++ b/source/util/src/tlockfree.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tlockfree.h" #define TD_RWLATCH_WRITE_FLAG 0x40000000 @@ -22,7 +22,7 @@ void taosInitRWLatch(SRWLatch *pLatch) { *pLatch = 0; } void taosWLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; - int nLoops = 0; + int32_t nLoops = 0; // Set write flag while (1) { @@ -57,7 +57,7 @@ void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); } void taosRLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; - int nLoops = 0; + int32_t nLoops = 0; while (1) { oLatch = atomic_load_32(pLatch); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2ed8d6e347357f458e3035b6e877717cda669d50..d821e0440b814f28e6e9274261f21fffac10b00d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "tutil.h" #include "tlog.h" +#include "tutil.h" #define LOG_MAX_LINE_SIZE (1000) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 10) @@ -113,7 +113,7 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(const char *logName, int maxFiles) { +int32_t taosInitLog(const char *logName, int32_t maxFiles) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; osUpdate(); diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 8b7f55809baa1eabbd10be49d22b4f44d2db8c90..6349ab170c3df30abd94f8b978f16aa598bc22da 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -13,12 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tlog.h" +#define _DEFAULT_SOURCE #include "tlosertree.h" #include "taoserror.h" - - +#include "tlog.h" // Set the initial value of the multiway merge tree. static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { @@ -33,10 +31,12 @@ static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { } } -int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, __merge_compare_fn_t compareFn) { +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, + __merge_compare_fn_t compareFn) { int32_t totalEntries = numOfSources << 1u; - SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + SMultiwayMergeTreeInfo* pTreeInfo = + (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -88,7 +88,7 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { return; } - int32_t parentId = idx >> 1; + int32_t parentId = idx >> 1; STreeNode kLeaf = pTree->pNode[idx]; while (parentId > 0) { diff --git a/source/util/src/mallocator.c b/source/util/src/tmallocator.c similarity index 98% rename from source/util/src/mallocator.c rename to source/util/src/tmallocator.c index a56fbfa5976dfcaaad681ecfda47ad8a1d415a6e..057b908a583f297227982a8f101ab97a7ebe891a 100644 --- a/source/util/src/mallocator.c +++ b/source/util/src/tmallocator.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "mallocator.h" +#define _DEFAULT_SOURCE +#include "tmallocator.h" /* ------------------------ HEAP ALLOCATOR ------------------------ */ #if 0 diff --git a/source/util/src/tmd5.c b/source/util/src/tmd5.c index 807f3c8122b31cea64e896aa730529b9af595cf3..2ea4415d7f4327c7a8806b3daebb644bd9a40738 100644 --- a/source/util/src/tmd5.c +++ b/source/util/src/tmd5.c @@ -33,7 +33,7 @@ *********************************************************************** */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tmd5.h" /* forward declaration */ @@ -98,13 +98,13 @@ void tMD5Init(T_MD5_CTX *mdContext) { account for the presence of each of the characters inBuf[0..inLen-1] in the message whose digest is being computed. */ -void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { - uint32_t in[16]; - int mdi; - unsigned int i, ii; +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, uint32_t inLen) { + uint32_t in[16]; + uint32_t mdi; + uint32_t i, ii; /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + mdi = (uint32_t)((mdContext->i[0] >> 3) & 0x3F); /* update number of bits */ if ((mdContext->i[0] + ((uint32_t)inLen << 3)) < mdContext->i[0]) mdContext->i[1]++; @@ -130,17 +130,17 @@ void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { ends with the desired message digest in mdContext->digest[0...15]. */ void tMD5Final(T_MD5_CTX *mdContext) { - uint32_t in[16]; - int mdi; - unsigned int i, ii; - unsigned int padLen; + uint32_t in[16]; + uint32_t mdi; + uint32_t i, ii; + uint32_t padLen; /* save number of bits */ in[14] = mdContext->i[0]; in[15] = mdContext->i[1]; /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + mdi = (uint32_t)((mdContext->i[0] >> 3) & 0x3F); /* pad out to 56 mod 64 */ padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index f980a05629f5504d14bdf168b0d753d42f8b4c57..1fc9bfc7ab458dfda7d1e74123bbea2366a70e38 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -13,22 +13,23 @@ * along with this program. If not, see . */ -#include "tlog.h" +#define _DEFAULT_SOURCE #include "tmempool.h" +#include "tlog.h" #include "tutil.h" typedef struct { - int numOfFree; /* number of free slots */ - int first; /* the first free slot */ - int numOfBlock; /* the number of blocks */ - int blockSize; /* block size in bytes */ - int * freeList; /* the index list */ - char * pool; /* the actual mem block */ + int32_t numOfFree; /* number of free slots */ + int32_t first; /* the first free slot */ + int32_t numOfBlock; /* the number of blocks */ + int32_t blockSize; /* block size in bytes */ + int32_t *freeList; /* the index list */ + char *pool; /* the actual mem block */ pthread_mutex_t mutex; } pool_t; -mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { - int i; +mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { + int32_t i; pool_t *pool_p; if (numOfBlock <= 1 || blockSize <= 1) { @@ -47,7 +48,7 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; pool_p->pool = (char *)malloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int *)malloc(sizeof(int) * (size_t)numOfBlock); + pool_p->freeList = (int32_t *)malloc(sizeof(int32_t) * (size_t)numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); @@ -69,7 +70,7 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { } char *taosMemPoolMalloc(mpool_h handle) { - char * pos = NULL; + char *pos = NULL; pool_t *pool_p = (pool_t *)handle; pthread_mutex_lock(&(pool_p->mutex)); @@ -88,20 +89,20 @@ char *taosMemPoolMalloc(mpool_h handle) { } void taosMemPoolFree(mpool_h handle, char *pMem) { - int index; + int32_t index; pool_t *pool_p = (pool_t *)handle; if (pMem == NULL) return; - index = (int)(pMem - pool_p->pool) % pool_p->blockSize; + index = (int32_t)(pMem - pool_p->pool) % pool_p->blockSize; if (index != 0) { uError("invalid free address:%p\n", pMem); return; } - index = (int)((pMem - pool_p->pool) / pool_p->blockSize); + index = (int32_t)((pMem - pool_p->pool) / pool_p->blockSize); if (index < 0 || index >= pool_p->numOfBlock) { - uError("mempool: error, invalid address:%p\n", pMem); + uError("mempool: error, invalid address:%p", pMem); return; } diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index a7e43cebf9c813c028722a4849e841ae18c0baf4..45c38165a11c47717b740fd74e5f43a5947c9406 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -1,109 +1,64 @@ -#include "tlog.h" +#define _DEFAULT_SOURCE #include "tpagedbuf.h" #include "taoserror.h" #include "tcompression.h" #include "thash.h" +#include "tlog.h" -#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) +#define GET_DATA_PAYLOAD(_p) ((char*)(_p)->pData + POINTER_BYTES) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) -typedef struct SFreeListItem { - int32_t offset; - int32_t len; -} SFreeListItem; - typedef struct SPageDiskInfo { - int64_t offset; - int32_t length; -} SPageDiskInfo; - -typedef struct SPageInfo { - SListNode* pn; // point to list node - void* pData; - int64_t offset; - int32_t pageId; - int32_t length:30; - bool used:1; // set current page is in used - bool dirty:1; // set current buffer page is dirty or not -} SPageInfo; - -typedef struct SDiskbasedBuf { + int64_t offset; + int32_t length; +} SPageDiskInfo, SFreeListItem; + +struct SPageInfo { + SListNode* pn; // point to list node struct + void* pData; + int64_t offset; + int32_t pageId; + int32_t length : 29; + bool used : 1; // set current page is in used + bool dirty : 1; // set current buffer page is dirty or not +}; + +struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; - uint64_t fileSize; // disk file size + uint64_t fileSize; // disk file size TdFilePtr pFile; - int32_t allocateId; // allocated page id - char* path; // file path - int32_t pageSize; // current used page size - int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* groupSet; // id hash table + int32_t allocateId; // allocated page id + char* path; // file path + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory + SList* freePgList; // free page list + SHashObj* groupSet; // id hash table, todo remove it SHashObj* all; SList* lruList; - void* emptyDummyIdList; // dummy id list - void* assistBuf; // assistant buffer for compress/decompress data - SArray* pFree; // free area in file - bool comp; // compressed before flushed to disk - uint64_t nextPos; // next page flush position - - uint64_t qId; // for debug purpose - bool printStatis; // Print statistics info when closing this buffer. + void* emptyDummyIdList; // dummy id list + void* assistBuf; // assistant buffer for compress/decompress data + SArray* pFree; // free area in file + bool comp; // compressed before flushed to disk + uint64_t nextPos; // next page flush position + + uint64_t qId; // for debug purpose + bool printStatis; // Print statistics info when closing this buffer. SDiskbasedBufStatis statis; -} SDiskbasedBuf; - -static void printStatisData(const SDiskbasedBuf* pBuf); - - int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) { - *pBuf = calloc(1, sizeof(SDiskbasedBuf)); - - SDiskbasedBuf* pResBuf = *pBuf; - if (pResBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = 0; // all pages are in buffer in the first place - pResBuf->totalBufSize = 0; - pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. - pResBuf->allocateId = -1; - pResBuf->comp = true; - pResBuf->pFile = NULL; - pResBuf->qId = qId; - pResBuf->fileSize = 0; - - // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); - - pResBuf->lruList = tdListNew(POINTER_BYTES); - - // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - -// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, -// pResBuf->inMemPages, pResBuf->path); - - return TSDB_CODE_SUCCESS; -} +}; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { // pBuf->file = fopen(pBuf->path, "wb+"); pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); if (pBuf->pFile == NULL) { -// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); + // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } return TSDB_CODE_SUCCESS; } -static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing +static char* doCompressData(void* data, int32_t srcSize, int32_t* dst, SDiskbasedBuf* pBuf) { // do nothing if (!pBuf->comp) { *dst = srcSize; return data; @@ -115,13 +70,13 @@ static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbase return data; } -static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing +static char* doDecompressData(void* data, int32_t srcSize, int32_t* dst, SDiskbasedBuf* pBuf) { // do nothing if (!pBuf->comp) { *dst = srcSize; return data; } - *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0); + *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize, ONE_STAGE_COMP, NULL, 0); if (*dst > 0) { memcpy(data, pBuf->assistBuf, *dst); } @@ -135,12 +90,12 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) { int32_t offset = -1; size_t num = taosArrayGetSize(pBuf->pFree); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { SFreeListItem* pi = taosArrayGet(pBuf->pFree, i); - if (pi->len >= size) { + if (pi->length >= size) { offset = pi->offset; pi->offset += (int32_t)size; - pi->len -= (int32_t)size; + pi->length -= (int32_t)size; return offset; } @@ -151,87 +106,106 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) { } } +static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; } + +static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + 2; } + +/** + * +--------------------------+-------------------+--------------+ + * | PTR to SPageInfo (8bytes)| Payload (PageSize)| 2 Extra Bytes| + * +--------------------------+-------------------+--------------+ + * @param pBuf + * @param pg + * @return + */ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { assert(!pg->used && pg->pData != NULL); - int32_t size = -1; + int32_t size = pBuf->pageSize; char* t = NULL; if (pg->offset == -1 || pg->dirty) { - SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - t = doCompressData(pPage->data, pBuf->pageSize, &size, pBuf); + void* payload = GET_DATA_PAYLOAD(pg); + t = doCompressData(payload, pBuf->pageSize, &size, pBuf); + assert(size >= 0); } // this page is flushed to disk for the first time - if (pg->offset == -1) { - assert(pg->dirty == true); + if (pg->dirty) { + if (pg->offset == -1) { + assert(pg->dirty == true); - pg->offset = allocatePositionInFile(pBuf, size); - pBuf->nextPos += size; + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; - int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); - if (ret != size) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + ret = (int32_t)taosWriteFile(pBuf->pFile, t, size); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - if (pBuf->fileSize < pg->offset + size) { - pBuf->fileSize = pg->offset + size; - } + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } - pBuf->statis.flushBytes += size; - pBuf->statis.flushPages += 1; - } else if (pg->dirty) { - // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing - if (pg->length < size) { - // 1. add current space to free list - SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset}; - taosArrayPush(pBuf->pFree, &dinfo); + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } else { + // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing + if (pg->length < size) { + // 1. add current space to free list + SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset}; + taosArrayPush(pBuf->pFree, &dinfo); + + // 2. allocate new position, and update the info + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; + } - // 2. allocate new position, and update the info - pg->offset = allocatePositionInFile(pBuf, size); - pBuf->nextPos += size; - } + // 3. write to disk. + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - // 3. write to disk. - int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + ret = (int32_t)taosWriteFile(pBuf->pFile, t, size); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - ret = (int32_t) taosWriteFile(pBuf->pFile, t, size); - if (ret != size) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } - if (pBuf->fileSize < pg->offset + size) { - pBuf->fileSize = pg->offset + size; + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } + } else { // NOTE: the size may be -1, the this recycle page has not been flushed to disk yet. + size = pg->length; + if (size == -1) { + printf("----\n"); } - - pBuf->statis.flushBytes += size; - pBuf->statis.flushPages += 1; } - char* pDataBuf = pg->pData; - memset(pDataBuf, 0, pBuf->pageSize + sizeof(SFilePage)); + ASSERT(size > 0 || (pg->offset == -1 && pg->length == -1)); - pg->pData = NULL; // this means the data is not in buffer - pg->length = size; - pg->dirty = false; + char* pDataBuf = pg->pData; + memset(pDataBuf, 0, getAllocPageSize(pBuf->pageSize)); + pg->length = size; // on disk size return pDataBuf; } static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); + assert(((int64_t)pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { @@ -240,7 +214,11 @@ static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { } } - return doFlushPageToDisk(pBuf, pg); + char* p = doFlushPageToDisk(pBuf, pg); + setPageNotInBuf(pg); + pg->dirty = false; + + return p; } // load file block data in disk @@ -251,8 +229,8 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { return ret; } - SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)taosReadFile(pBuf->pFile, pPage->data, pg->length); + void* pPage = (void*)GET_DATA_PAYLOAD(pg); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -262,12 +240,12 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pBuf->statis.loadPages += 1; int32_t fullSize = 0; - doDecompressData(pPage->data, pg->length, &fullSize, pBuf); + doDecompressData(pPage, pg->length, &fullSize, pBuf); return 0; } static SIDList addNewGroup(SDiskbasedBuf* pBuf, int32_t groupId) { - assert(taosHashGet(pBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); + assert(taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t)) == NULL); SArray* pa = taosArrayInit(1, POINTER_BYTES); int32_t ret = taosHashPut(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); @@ -283,21 +261,21 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag if (p == NULL) { // it is a new group id list = addNewGroup(pBuf, groupId); } else { - list = (SIDList) (*p); + list = (SIDList)(*p); } pBuf->numOfPages += 1; - SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; + SPageInfo* ppi = malloc(sizeof(SPageInfo)); ppi->pageId = pageId; - ppi->pData = NULL; + ppi->pData = NULL; ppi->offset = -1; ppi->length = -1; - ppi->used = true; - ppi->pn = NULL; + ppi->used = true; + ppi->pn = NULL; - return *(SPageInfo**) taosArrayPush(list, &ppi); + return *(SPageInfo**)taosArrayPush(list, &ppi); } static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { @@ -305,22 +283,33 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { tdListInitIter(pBuf->lruList, &iter, TD_LIST_BACKWARD); SListNode* pn = NULL; - while((pn = tdListNext(&iter)) != NULL) { - assert(pn != NULL); - - SPageInfo* pageInfo = *(SPageInfo**) pn->data; + while ((pn = tdListNext(&iter)) != NULL) { + SPageInfo* pageInfo = *(SPageInfo**)pn->data; assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); if (!pageInfo->used) { + // printf("%d is chosen\n", pageInfo->pageId); break; + } else { + // printf("page %d is used, dirty:%d\n", pageInfo->pageId, pageInfo->dirty); } } + // int32_t pos = listNEles(pBuf->lruList); + // SListIter iter1 = {0}; + // tdListInitIter(pBuf->lruList, &iter1, TD_LIST_BACKWARD); + // SListNode* pn1 = NULL; + // while((pn1 = tdListNext(&iter1)) != NULL) { + // SPageInfo* pageInfo = *(SPageInfo**) pn1->data; + // printf("page %d is used, dirty:%d, pos:%d\n", pageInfo->pageId, pageInfo->dirty, pos - 1); + // pos -= 1; + // } + return pn; } static char* evacOneDataPage(SDiskbasedBuf* pBuf) { - char* bufPage = NULL; + char* bufPage = NULL; SListNode* pn = getEldestUnrefedPage(pBuf); // all pages are referenced by user, try to allocate new space @@ -331,12 +320,12 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { // increase by 50% of previous mem pages pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f); -// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev, -// pBuf->inMemPages, pBuf->pageSize); + // qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev, + // pBuf->inMemPages, pBuf->pageSize); } else { tdListPopNode(pBuf->lruList, pn); - SPageInfo* d = *(SPageInfo**) pn->data; + SPageInfo* d = *(SPageInfo**)pn->data; assert(d->pn == pn); d->pn = NULL; @@ -348,22 +337,71 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { return bufPage; } -static void lruListPushFront(SList *pList, SPageInfo* pi) { +static void lruListPushFront(SList* pList, SPageInfo* pi) { tdListPrepend(pList, &pi); SListNode* front = tdListGetHead(pList); pi->pn = front; } -static void lruListMoveToFront(SList *pList, SPageInfo* pi) { +static void lruListMoveToFront(SList* pList, SPageInfo* pi) { tdListPopNode(pList, pi->pn); tdListPrependNode(pList, pi->pn); } -static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { - return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage); +static SPageInfo* getPageInfoFromPayload(void* page) { + int32_t offset = offsetof(SPageInfo, pData); + char* p = page - offset; + + SPageInfo* ppi = ((SPageInfo**)p)[0]; + return ppi; } -SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, + const char* dir) { + *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + + SDiskbasedBuf* pPBuf = *pBuf; + if (pPBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pPBuf->pageSize = pagesize; + pPBuf->numOfPages = 0; // all pages are in buffer in the first place + pPBuf->totalBufSize = 0; + pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. + pPBuf->allocateId = -1; + pPBuf->comp = true; + pPBuf->pFile = NULL; + pPBuf->qId = qId; + pPBuf->fileSize = 0; + pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); + pPBuf->freePgList = tdListNew(POINTER_BYTES); + + // at least more than 2 pages must be in memory + assert(inMemBufSize >= pagesize * 2); + + pPBuf->lruList = tdListNew(POINTER_BYTES); + + // init id hash table + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + pPBuf->groupSet = taosHashInit(10, fn, true, false); + pPBuf->assistBuf = malloc(pPBuf->pageSize + 2); // EXTRA BYTES + pPBuf->all = taosHashInit(10, fn, true, false); + + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(dir, "paged-buf", path); + pPBuf->path = strdup(path); + + pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); + + // qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, + // pPBuf->pageSize, + // pPBuf->inMemPages, pPBuf->path); + + return TSDB_CODE_SUCCESS; +} + +void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { pBuf->statis.getPages += 1; char* availablePage = NULL; @@ -372,23 +410,34 @@ SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) // Failed to allocate a new buffer page, and there is an error occurs. if (availablePage == NULL) { + assert(0); return NULL; } } - // register new id in this group - *pageId = (++pBuf->allocateId); - - // register page id info - SPageInfo* pi = registerPage(pBuf, groupId, *pageId); + SPageInfo* pi = NULL; + if (listNEles(pBuf->freePgList) != 0) { + SListNode* pItem = tdListPopHead(pBuf->freePgList); + pi = *(SPageInfo**)pItem->data; + pi->used = true; + *pageId = pi->pageId; + tfree(pItem); + } else { // create a new pageinfo + // register new id in this group + *pageId = (++pBuf->allocateId); + + // register page id info + pi = registerPage(pBuf, groupId, *pageId); + + // add to hash map + taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); + pBuf->totalBufSize += pBuf->pageSize; + } // add to LRU list assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0); lruListPushFront(pBuf->lruList, pi); - // add to hash map - taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); - // allocate buf if (availablePage == NULL) { pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. @@ -396,37 +445,32 @@ SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) pi->pData = availablePage; } - pBuf->totalBufSize += pBuf->pageSize; - ((void**)pi->pData)[0] = pi; - pi->used = true; - - return (void *)(GET_DATA_PAYLOAD(pi)); + return (void*)(GET_DATA_PAYLOAD(pi)); } -SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { +void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { assert(pBuf != NULL && id >= 0); pBuf->statis.getPages += 1; SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t)); assert(pi != NULL && *pi != NULL); - if ((*pi)->pData != NULL) { // it is in memory + if ((*pi)->pData != NULL) { // it is in memory // no need to update the LRU list if only one page exists if (pBuf->numOfPages == 1) { (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); + return (void*)(GET_DATA_PAYLOAD(*pi)); } - SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); + SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); assert(*pInfo == *pi); lruListMoveToFront(pBuf->lruList, (*pi)); (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); - - } else { // not in memory + return (void*)(GET_DATA_PAYLOAD(*pi)); + } else { // not in memory assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0); char* availablePage = NULL; @@ -443,6 +487,7 @@ SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { (*pi)->pData = availablePage; } + // set the ptr to the new SPageInfo ((void**)((*pi)->pData))[0] = (*pi); lruListPushFront(pBuf->lruList, *pi); @@ -453,27 +498,24 @@ SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { return NULL; } - return (void *)(GET_DATA_PAYLOAD(*pi)); + return (void*)(GET_DATA_PAYLOAD(*pi)); } } void releaseBufPage(SDiskbasedBuf* pBuf, void* page) { assert(pBuf != NULL && page != NULL); - int32_t offset = offsetof(SPageInfo, pData); - char* p = page - offset; - - SPageInfo* ppi = ((SPageInfo**) p)[0]; + SPageInfo* ppi = getPageInfoFromPayload(page); releaseBufPageInfo(pBuf, ppi); } void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { - assert(pi->pData != NULL && pi->used); + assert(pi->pData != NULL && pi->used == true); pi->used = false; pBuf->statis.releasePages += 1; } -size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); } +size_t getNumOfBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); } size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; } @@ -484,42 +526,46 @@ SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId) { if (p == NULL) { // it is a new group id return pBuf->emptyDummyIdList; } else { - return (SArray*) (*p); + return (SArray*)(*p); } } -void destroyResultBuf(SDiskbasedBuf* pBuf) { +void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (pBuf == NULL) { return; } - printStatisData(pBuf); + dBufPrintStatis(pBuf); if (pBuf->pFile != NULL) { - uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", - pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); + uDebug( + "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " + "size:%.2f Kb, %" PRIx64 "\n", + pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); - taosCloseFile(&pBuf->pFile); + taosCloseFile(&pBuf->pFile); } else { - uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); + uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId); } // print the statistics information { - SDiskbasedBufStatis *ps = &pBuf->statis; - uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n" - , ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages - , ps->loadBytes/(1024.0 * ps->loadPages)); + SDiskbasedBufStatis* ps = &pBuf->statis; + uDebug( + "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f " + "Kb\n", + ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f, + ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages)); } remove(pBuf->path); tfree(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); - while(p) { + while (p) { size_t n = taosArrayGetSize(*p); - for(int32_t i = 0; i < n; ++i) { + for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); tfree(pi->pData); tfree(pi); @@ -530,7 +576,11 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { } tdListFree(pBuf->lruList); + tdListFree(pBuf->freePgList); + taosArrayDestroy(pBuf->emptyDummyIdList); + taosArrayDestroy(pBuf->pFree); + taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); @@ -539,7 +589,7 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { } SPageInfo* getLastPageInfo(SIDList pList) { - size_t size = taosArrayGetSize(pList); + size_t size = taosArrayGetSize(pList); SPageInfo* pPgInfo = taosArrayGetP(pList, size - 1); return pPgInfo; } @@ -549,35 +599,38 @@ int32_t getPageId(const SPageInfo* pPgInfo) { return pPgInfo->pageId; } -int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { - return pBuf->pageSize; -} - -int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { - return pBuf->inMemPages; -} +int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { return pBuf->pageSize; } -bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { - return pBuf->fileSize == 0; -} +int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { return pBuf->inMemPages; } -void setBufPageDirty(SFilePage* pPage, bool dirty) { - int32_t offset = offsetof(SPageInfo, pData); // todo extract method - char* p = (char*)pPage - offset; +bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { return pBuf->fileSize == 0; } - SPageInfo* ppi = ((SPageInfo**) p)[0]; +void setBufPageDirty(void* pPage, bool dirty) { + SPageInfo* ppi = getPageInfoFromPayload(pPage); ppi->dirty = dirty; } -void printStatisBeforeClose(SDiskbasedBuf* pBuf) { - pBuf->printStatis = true; -} +void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp) { pBuf->comp = comp; } + +void dBufSetBufPageRecycled(SDiskbasedBuf* pBuf, void* pPage) { + SPageInfo* ppi = getPageInfoFromPayload(pPage); + + ppi->used = false; + ppi->dirty = false; -SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { - return pBuf->statis; + // add this pageinfo into the free page info list + SListNode* pNode = tdListPopNode(pBuf->lruList, ppi->pn); + tfree(ppi->pData); + tfree(pNode); + + tdListAppend(pBuf->freePgList, &ppi); } -void printStatisData(const SDiskbasedBuf* pBuf) { +void dBufSetPrintInfo(SDiskbasedBuf* pBuf) { pBuf->printStatis = true; } + +SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { return pBuf->statis; } + +void dBufPrintStatis(const SDiskbasedBuf* pBuf) { if (!pBuf->printStatis) { return; } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 0f6e2610c5df6ce7a3b674c0187889f095970ca6..000352203348ec8ea0ff3d587c8de89557d994dd 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -30,19 +30,19 @@ typedef struct STaosQueue { int32_t itemSize; int32_t numOfItems; int32_t threadId; - STaosQnode * head; - STaosQnode * tail; - STaosQueue * next; // for queue set - STaosQset * qset; // for queue set - void * ahandle; // for queue set + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set FItem itemFp; FItems itemsFp; pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { - STaosQueue * head; - STaosQueue * current; + STaosQueue *head; + STaosQueue *current; pthread_mutex_t mutex; int32_t numOfQueues; int32_t numOfItems; @@ -82,7 +82,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset * qset; + STaosQset *qset; pthread_mutex_lock(&queue->mutex); STaosQnode *pNode = queue->head; diff --git a/source/util/src/tref.c b/source/util/src/tref.c index ca4388ec26f7da618e2878d4ecc79d3bb9545376..a9f6c21bf80ad48f34c895e25c46074fe013d7cd 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" - +#define _DEFAULT_SOURCE +#include "tref.h" #include "taoserror.h" #include "tlog.h" #include "tutil.h" @@ -25,49 +25,48 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; // previous node - struct SRefNode *next; // next node - void *p; // pointer to resource protected, - int64_t rid; // reference ID - int32_t count; // number of references - int removed; // 1: removed + struct SRefNode *prev; // previous node + struct SRefNode *next; // next node + void *p; // pointer to resource protected, + int64_t rid; // reference ID + int32_t count; // number of references + int32_t removed; // 1: removed } SRefNode; - + typedef struct { - SRefNode **nodeList; // array of SRefNode linked list - int state; // 0: empty, 1: active; 2: deleted - int rsetId; // refSet ID, global unique - int64_t rid; // increase by one for each new reference - int max; // mod - int32_t count; // total number of SRefNodes in this set + SRefNode **nodeList; // array of SRefNode linked list + int32_t state; // 0: empty, 1: active; 2: deleted + int32_t rsetId; // refSet ID, global unique + int64_t rid; // increase by one for each new reference + int32_t max; // mod + int32_t count; // total number of SRefNodes in this set int64_t *lockedBy; - void (*fp)(void *); + void (*fp)(void *); } SRefSet; static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tsRefMutex; -static int tsRefSetNum = 0; -static int tsNextId = 0; - -static void taosInitRefModule(void); -static void taosLockList(int64_t *lockedBy); -static void taosUnlockList(int64_t *lockedBy); -static void taosIncRsetCount(SRefSet *pSet); -static void taosDecRsetCount(SRefSet *pSet); -static int taosDecRefCount(int rsetId, int64_t rid, int remove); - -int taosOpenRef(int max, void (*fp)(void *)) -{ +static int32_t tsRefSetNum = 0; +static int32_t tsNextId = 0; + +static void taosInitRefModule(void); +static void taosLockList(int64_t *lockedBy); +static void taosUnlockList(int64_t *lockedBy); +static void taosIncRsetCount(SRefSet *pSet); +static void taosDecRsetCount(SRefSet *pSet); +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove); + +int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { SRefNode **nodeList; SRefSet *pSet; int64_t *lockedBy; - int i, rsetId; + int32_t i, rsetId; pthread_once(&tsRefModuleInit, taosInitRefModule); nodeList = calloc(sizeof(SRefNode *), (size_t)max); - if (nodeList == NULL) { + if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } @@ -83,7 +82,7 @@ int taosOpenRef(int max, void (*fp)(void *)) for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; - if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId + if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; } @@ -103,8 +102,8 @@ int taosOpenRef(int max, void (*fp)(void *)) uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { rsetId = TSDB_CODE_REF_FULL; - free (nodeList); - free (lockedBy); + free(nodeList); + free(lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } @@ -113,10 +112,9 @@ int taosOpenRef(int max, void (*fp)(void *)) return rsetId; } -int taosCloseRef(int rsetId) -{ - SRefSet *pSet; - int deleted = 0; +int32_t taosCloseRef(int32_t rsetId) { + SRefSet *pSet; + int32_t deleted = 0; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { uTrace("rsetId:%d is invalid, out of range", rsetId); @@ -143,9 +141,8 @@ int taosCloseRef(int rsetId) return 0; } -int64_t taosAddRef(int rsetId, void *p) -{ - int hash; +int64_t taosAddRef(int32_t rsetId, void *p) { + int32_t hash; SRefNode *pNode; SRefSet *pSet; int64_t rid = 0; @@ -173,7 +170,7 @@ int64_t taosAddRef(int rsetId, void *p) rid = atomic_add_fetch_64(&pSet->rid, 1); hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode->p = p; pNode->rid = rid; @@ -186,26 +183,22 @@ int64_t taosAddRef(int rsetId, void *p) uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count); - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); return rid; } -int taosRemoveRef(int rsetId, int64_t rid) -{ - return taosDecRefCount(rsetId, rid, 1); -} +int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosAcquireRef(int rsetId, int64_t rid) -{ - int hash; +void *taosAcquireRef(int32_t rsetId, int64_t rid) { + int32_t hash; SRefNode *pNode; SRefSet *pSet; void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - //uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + // uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } @@ -226,7 +219,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) } hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; @@ -252,20 +245,17 @@ void *taosAcquireRef(int rsetId, int64_t rid) uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid); } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); taosDecRsetCount(pSet); return p; } -int taosReleaseRef(int rsetId, int64_t rid) -{ - return taosDecRefCount(rsetId, rid, 0); -} +int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosIterateRef(int rsetId, int64_t rid) { +void *taosIterateRef(int32_t rsetId, int64_t rid) { SRefNode *pNode = NULL; SRefSet *pSet; @@ -293,10 +283,10 @@ void *taosIterateRef(int rsetId, int64_t rid) { do { newP = NULL; - int hash = 0; + int32_t hash = 0; if (rid > 0) { hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; while (pNode) { @@ -307,7 +297,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { if (pNode == NULL) { uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); taosDecRsetCount(pSet); return NULL; } @@ -320,14 +310,14 @@ void *taosIterateRef(int rsetId, int64_t rid) { pNode = pNode->next; } if (pNode == NULL) { - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); hash++; } } if (pNode == NULL) { for (; hash < pSet->max; ++hash) { - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; if (pNode) { // check first place @@ -337,14 +327,14 @@ void *taosIterateRef(int rsetId, int64_t rid) { } if (pNode) break; } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); } } if (pNode) { pNode->count++; // acquire it newP = pNode->p; - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); } else { uTrace("rsetId:%d the list is over", rsetId); @@ -359,22 +349,21 @@ void *taosIterateRef(int rsetId, int64_t rid) { return newP; } -int taosListRef() { +int32_t taosListRef() { SRefSet *pSet; SRefNode *pNode; - int num = 0; + int32_t num = 0; pthread_mutex_lock(&tsRefMutex); - for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { + for (int32_t i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; - if (pSet->state == TSDB_REF_STATE_EMPTY) - continue; + if (pSet->state == TSDB_REF_STATE_EMPTY) continue; uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); - for (int j=0; j < pSet->max; ++j) { + for (int32_t j = 0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; while (pNode) { @@ -390,12 +379,12 @@ int taosListRef() { return num; } -static int taosDecRefCount(int rsetId, int64_t rid, int remove) { - int hash; +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { + int32_t hash; SRefSet *pSet; SRefNode *pNode; - int released = 0; - int code = 0; + int32_t released = 0; + int32_t code = 0; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid); @@ -417,12 +406,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; while (pNode) { - if (pNode->rid == rid) - break; + if (pNode->rid == rid) break; pNode = pNode->next; } @@ -437,13 +425,13 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } else { pSet->nodeList[hash] = pNode->next; } - + if (pNode->next) { pNode->next->prev = pNode->prev; } released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); @@ -451,10 +439,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { code = -1; } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); if (released) { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, + pNode); (*pSet->fp)(pNode->p); free(pNode); @@ -466,7 +455,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { static void taosLockList(int64_t *lockedBy) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { sched_yield(); @@ -481,9 +470,7 @@ static void taosUnlockList(int64_t *lockedBy) { } } -static void taosInitRefModule(void) { - pthread_mutex_init(&tsRefMutex, NULL); -} +static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); @@ -512,4 +499,3 @@ static void taosDecRsetCount(SRefSet *pSet) { pthread_mutex_unlock(&tsRefMutex); } - diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 7db5abdd24370fdf37236c2c8a616e84149b4a53..740e742bad5e9b9f9e50194bcc02aadc1146cba3 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -13,34 +13,35 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "tsched.h" #include "tdef.h" -#include "tutil.h" #include "tlog.h" -#include "tsched.h" #include "ttimer.h" +#include "tutil.h" -#define DUMP_SCHEDULER_TIME_WINDOW 30000 //every 30sec, take a snap shot of task queue. +#define DUMP_SCHEDULER_TIME_WINDOW 30000 // every 30sec, take a snap shot of task queue. typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; pthread_mutex_t queueMutex; - int fullSlot; - int emptySlot; - int queueSize; - int numOfThreads; - pthread_t * qthread; - SSchedMsg * queue; + int32_t fullSlot; + int32_t emptySlot; + int32_t queueSize; + int32_t numOfThreads; + pthread_t *qthread; + SSchedMsg *queue; bool stop; - void* pTmrCtrl; - void* pTimer; + void *pTmrCtrl; + void *pTimer; } SSchedQueue; static void *taosProcessSchedQueue(void *param); -static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); +static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); -void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { +void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *label) { SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { uError("%s: no enough memory for pSched", label); @@ -62,7 +63,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pSched->queueSize = queueSize; - tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow + tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow pSched->fullSlot = 0; pSched->emptySlot = 0; @@ -73,7 +74,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { return NULL; } - if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { + if (tsem_init(&pSched->emptySem, 0, (uint32_t)pSched->queueSize) != 0) { uError("init %s:empty semaphore failed(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); return NULL; @@ -86,11 +87,11 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pSched->stop = false; - for (int i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + int32_t code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); pthread_attr_destroy(&attr); if (code != 0) { uError("%s: failed to create rpc thread(%s)", label, strerror(errno)); @@ -105,8 +106,8 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { return (void *)pSched; } -void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { - SSchedQueue* pSched = taosInitScheduler(queueSize, numOfThreads, label); +void *taosInitSchedulerWithInfo(int32_t queueSize, int32_t numOfThreads, const char *label, void *tmrCtrl) { + SSchedQueue *pSched = taosInitScheduler(queueSize, numOfThreads, label); if (tmrCtrl != NULL && pSched != NULL) { pSched->pTmrCtrl = tmrCtrl; @@ -119,7 +120,7 @@ void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *lab void *taosProcessSchedQueue(void *scheduler) { SSchedMsg msg; SSchedQueue *pSched = (SSchedQueue *)scheduler; - int ret = 0; + int32_t ret = 0; char name[16] = {0}; snprintf(name, tListLen(name), "%s-taskQ", pSched->label); @@ -164,7 +165,7 @@ void *taosProcessSchedQueue(void *scheduler) { void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { SSchedQueue *pSched = (SSchedQueue *)queueScheduler; - int ret = 0; + int32_t ret = 0; if (pSched == NULL) { uError("sched is not ready, msg:%p is dropped", pMsg); @@ -200,12 +201,12 @@ void taosCleanUpScheduler(void *param) { if (pSched == NULL) return; pSched->stop = true; - for (int i = 0; i < pSched->numOfThreads; ++i) { - if (taosCheckPthreadValid(pSched->qthread[i])) { + for (int32_t i = 0; i < pSched->numOfThreads; ++i) { + if (taosCheckPthreadValid(pSched->qthread[i])) { tsem_post(&pSched->fullSem); } } - for (int i = 0; i < pSched->numOfThreads; ++i) { + for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { pthread_join(pSched->qthread[i], NULL); } @@ -214,14 +215,14 @@ void taosCleanUpScheduler(void *param) { tsem_destroy(&pSched->emptySem); tsem_destroy(&pSched->fullSem); pthread_mutex_destroy(&pSched->queueMutex); - + if (pSched->pTimer) { taosTmrStopA(&pSched->pTimer); } if (pSched->queue) free(pSched->queue); if (pSched->qthread) free(pSched->qthread); - free(pSched); // fix memory leak + free(pSched); // fix memory leak } // for debug purpose, dump the scheduler status every 1min. @@ -230,11 +231,11 @@ void taosDumpSchedulerStatus(void *qhandle, void *tmrId) { if (pSched == NULL || pSched->pTimer == NULL || pSched->pTimer != tmrId) { return; } - + int32_t size = ((pSched->emptySlot - pSched->fullSlot) + pSched->queueSize) % pSched->queueSize; if (size > 0) { uDebug("scheduler:%s, current tasks in queue:%d, task thread:%d", pSched->label, size, pSched->numOfThreads); } - + taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); } diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 1c325ca0a8519621eda72f40f520c8306aa6796e..6b89ed2c430f240a3b1a219df85b9b876f560c9b 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -14,13 +14,14 @@ * along with this program. If not, see . */ -#include "compare.h" +#define _DEFAULT_SOURCE #include "tskiplist.h" -#include "tutil.h" +#include "tcompare.h" #include "tlog.h" +#include "tutil.h" -static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); +static int32_t initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -31,10 +32,9 @@ static SSkipListNode *tSkipListNewNode(uint8_t level); static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); - -static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); -static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); -static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, @@ -138,21 +138,21 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool hasDup = false; - char * pKey = NULL; - char * pDataKey = NULL; - int compare = 0; + char *pKey = NULL; + char *pDataKey = NULL; + int32_t compare = 0; tSkipListWLock(pSkipList); - void* pData = iterate(iter); - if(pData == NULL) return; + void *pData = iterate(iter); + if (pData == NULL) return; // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); tSkipListPutImpl(pSkipList, pData, backward, false, hasDup); - for (int level = 0; level < pSkipList->maxLevel; level++) { + for (int32_t level = 0; level < pSkipList->maxLevel; level++) { forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level); } @@ -165,12 +165,12 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_MAX_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare > 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); } } else { SSkipListNode *px = pSkipList->pHead; - for (int i = pSkipList->maxLevel - 1; i >= 0; --i) { + for (int32_t i = pSkipList->maxLevel - 1; i >= 0; --i) { if (i < pSkipList->level) { // set new px if (forward[i] != pSkipList->pHead) { @@ -357,7 +357,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; - char * prev = NULL; + char *prev = NULL; while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); @@ -433,21 +433,21 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t return iter; } -static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_wrlock(pSkipList->lock); } return 0; } -static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_rdlock(pSkipList->lock); } return 0; } -static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_unlock(pSkipList->lock); } @@ -455,12 +455,12 @@ static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { } static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { - int compare = 0; + int32_t compare = 0; bool hasDupKey = false; - char * pDataKey = pSkipList->keyFn(pData); + char *pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = pSkipList->pTail; } } else { @@ -470,7 +470,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, pKey = SL_GET_MAX_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare >= 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = pSkipList->pTail; } @@ -481,7 +481,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, pKey = SL_GET_MIN_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare < 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i); } @@ -489,7 +489,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, } SSkipListNode *px = pSkipList->pTail; - for (int i = pSkipList->maxLevel - 1; i >= 0; --i) { + for (int32_t i = pSkipList->maxLevel - 1; i >= 0; --i) { if (i < pSkipList->level) { SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i); while (p != pSkipList->pHead) { @@ -532,7 +532,8 @@ static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) // Function must be called after calling tSkipListRemoveNodeImpl() function static void tSkipListCorrectLevel(SSkipList *pSkipList) { - while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + while (pSkipList->level > 0 && + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { pSkipList->level -= 1; } } @@ -636,7 +637,7 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ return pNode; } -static int initForwardBackwardPtr(SSkipList *pSkipList) { +static int32_t initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; // head info @@ -685,12 +686,12 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL pSkipList->insertHandleFn->args[1] = pNode->pData; pData = genericInvoke(pSkipList->insertHandleFn); } - if(pData) { + if (pData) { atomic_store_ptr(&(pNode->pData), pData); } } else { - //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! - if(pSkipList->insertHandleFn) { + // for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! + if (pSkipList->insertHandleFn) { pSkipList->insertHandleFn->args[0] = NULL; pSkipList->insertHandleFn->args[1] = NULL; genericInvoke(pSkipList->insertHandleFn); diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c deleted file mode 100644 index 30c8ec2b3aabdfeeb60c8716d63ae71dc0d00fb7..0000000000000000000000000000000000000000 --- a/source/util/src/tstep.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "tlog.h" -#include "taoserror.h" -#include "tstep.h" - -typedef struct { - char * name; - InitFp initFp; - CleanupFp cleanupFp; -} SStep; - -typedef struct SSteps { - int32_t cursize; - int32_t maxsize; - SStep * steps; - ReportFp reportFp; -} SSteps; - -SSteps *taosStepInit(int32_t maxsize, ReportFp fp) { - SSteps *steps = calloc(1, sizeof(SSteps)); - if (steps == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - steps->maxsize = maxsize; - steps->cursize = 0; - steps->steps = calloc(maxsize, sizeof(SStep)); - steps->reportFp = fp; - - return steps; -} - -int32_t taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp) { - if (steps == NULL) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - if (steps->cursize >= steps->maxsize) { - uError("failed to add step since up to the maxsize"); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - SStep step = {.name = name, .initFp = initFp, .cleanupFp = cleanupFp}; - steps->steps[steps->cursize++] = step; - return 0; -} - -static void taosStepCleanupImp(SSteps *steps, int32_t pos) { - for (int32_t s = pos; s >= 0; s--) { - SStep *step = steps->steps + s; - uDebug("step:%s will cleanup", step->name); - if (step->cleanupFp != NULL) { - (*step->cleanupFp)(); - } - } -} - -int32_t taosStepExec(SSteps *steps) { - if (steps == NULL) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - for (int32_t s = 0; s < steps->cursize; s++) { - SStep *step = steps->steps + s; - if (step->initFp == NULL) continue; - - if (steps->reportFp != NULL) { - (*steps->reportFp)(step->name, "start initialize"); - } - - int32_t code = (*step->initFp)(); - if (code != 0) { - uDebug("step:%s will cleanup", step->name); - taosStepCleanupImp(steps, s); - return code; - } - - uInfo("step:%s is initialized", step->name); - - if (steps->reportFp != NULL) { - (*steps->reportFp)(step->name, "initialize completed"); - } - } - - return 0; -} - -void taosStepCleanup(SSteps *steps) { - if (steps == NULL) return; - taosStepCleanupImp(steps, steps->cursize - 1); -} \ No newline at end of file diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index 230bff42f51abe19f6b8bb7b866db45f3475832c..f191f69986a840c520703979f503406ab94ba9bc 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -12,7 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" + +#define _DEFAULT_SOURCE #include "tstrbuild.h" void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 8c7a3ada05fab2ae2ab200b513957d93b9680e79..f9e28d7b6256bb61a51f8c76bc85da338c4f9d60 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -13,13 +13,9 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tthread.h" -#include "taoserror.h" -#include "tdef.h" -#include "tutil.h" -#include "tlog.h" -// create new thread pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_attr_t thattr; @@ -35,7 +31,6 @@ pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { return pthread; } -// destory thread bool taosDestoryThread(pthread_t* pthread) { if (pthread == NULL) return false; if (taosThreadRunning(pthread)) { @@ -47,10 +42,9 @@ bool taosDestoryThread(pthread_t* pthread) { return true; } -// thread running return true bool taosThreadRunning(pthread_t* pthread) { if (pthread == NULL) return false; - int ret = pthread_kill(*pthread, 0); + int32_t ret = pthread_kill(*pthread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 2c04603269c302bb6b85515f3ffb9a8b25fa5a3c..abb42ef28d6d0135cce72ac494e21b5b85d51418 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -13,12 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "ttimer.h" -#include "os.h" #include "taoserror.h" #include "tlog.h" #include "tsched.h" -#include "tutil.h" #define tmrFatal(...) \ { \ @@ -57,9 +56,9 @@ } \ } -#define TIMER_STATE_WAITING 0 -#define TIMER_STATE_EXPIRED 1 -#define TIMER_STATE_STOPPED 2 +#define TIMER_STATE_WAITING 0 +#define TIMER_STATE_EXPIRED 1 +#define TIMER_STATE_STOPPED 2 #define TIMER_STATE_CANCELED 3 typedef union _tmr_ctrl_t { @@ -118,9 +117,9 @@ static pthread_mutex_t tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; -static int numOfTmrCtrl = 0; +static int32_t numOfTmrCtrl = 0; -int taosTmrThreads = 1; +int32_t taosTmrThreads = 1; static uintptr_t nextTimerId = 0; static time_wheel_t wheels[] = { @@ -148,7 +147,7 @@ static void timerDecRef(tmr_obj_t* timer) { static void lockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { sched_yield(); @@ -322,7 +321,7 @@ static void addToExpired(tmr_obj_t* head) { } } -static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int mseconds, void* param, tmr_ctrl_t* ctrl) { +static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, tmr_ctrl_t* ctrl) { uintptr_t id = getNextTimerId(); timer->id = id; timer->state = TIMER_STATE_WAITING; @@ -346,7 +345,7 @@ static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int msecon return id; } -tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle) { +tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; if (ctrl == NULL || ctrl->label[0] == 0) { return NULL; @@ -361,10 +360,10 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle return (tmr_h)doStartTimer(timer, fp, mseconds, param, ctrl); } -static void taosTimerLoopFunc(int signo) { +static void taosTimerLoopFunc(int32_t signo) { int64_t now = taosGetMonotonicMs(); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. // expired timers are first add to this list, then move // to expired queue as a batch to improve performance. @@ -471,7 +470,7 @@ bool taosTmrStopA(tmr_h* timerId) { return ret; } -bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, tmr_h* pTmrId) { +bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; if (ctrl == NULL || ctrl->label[0] == 0) { return false; @@ -500,7 +499,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, // wait until there's no other reference to this timer, // so that we can reuse this timer safely. - for (int i = 1; atomic_load_8(&timer->refCount) > 1; ++i) { + for (int32_t i = 1; atomic_load_8(&timer->refCount) > 1; ++i) { if (i % 1000 == 0) { sched_yield(); } @@ -532,7 +531,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); int64_t now = taosGetMonotonicMs(); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); @@ -561,7 +560,7 @@ static void taosTmrModuleInit(void) { tmrDebug("timer module is initialized, number of threads: %d", taosTmrThreads); } -void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { +void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, const char* label) { const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); @@ -607,7 +606,7 @@ void taosTmrCleanUp(void* handle) { taosCleanUpScheduler(tmrQhandle); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; pthread_mutex_destroy(&wheel->mutex); free(wheel->slots); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 58a2e57f7c527cf8e75e8e0d3acf5beab0c4e6d7..7bd671a56ca42f9be79e123af734f1ba19ea83d6 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tdef.h" +#define _DEFAULT_SOURCE +#include "tutil.h" int32_t strdequote(char *z) { if (z == NULL) { @@ -47,38 +47,38 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -int32_t strRmquote(char *z, int32_t len){ - // delete escape character: \\, \', \" - char delim = z[0]; - if (delim != '\'' && delim != '\"') { - return len; - } - - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < len - 1; ++k) { - if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { - //match '_' self - } else { - z[j] = z[k + 1]; - cnt++; - j++; - k++; - continue; - } +int32_t strRmquote(char *z, int32_t len) { + // delete escape character: \\, \', \" + char delim = z[0]; + if (delim != '\'' && delim != '\"') { + return len; + } + + int32_t cnt = 0; + int32_t j = 0; + for (uint32_t k = 1; k < len - 1; ++k) { + if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { + if (z[k] == '\\' && z[k + 1] == '_') { + // match '_' self + } else { + z[j] = z[k + 1]; + cnt++; + j++; + k++; + continue; } - - z[j] = z[k]; - j++; } - - z[j] = 0; - - return len - 2 - cnt; + + z[j] = z[k]; + j++; + } + + z[j] = 0; + + return len - 2 - cnt; } -int32_t strndequote(char *dst, const char* z, int32_t len) { +int32_t strndequote(char *dst, const char *z, int32_t len) { assert(dst != NULL); if (z == NULL || len == 0) { return 0; @@ -90,7 +90,7 @@ int32_t strndequote(char *dst, const char* z, int32_t len) { while (z[i] != 0) { if (z[i] == quote) { if (z[i + 1] == quote) { - dst[j++] = (char) quote; + dst[j++] = (char)quote; i++; } else { dst[j++] = 0; @@ -139,7 +139,7 @@ size_t strtrim(char *z) { } else if (j != i) { z[i] = 0; } - + return i; } @@ -168,11 +168,11 @@ char **strsplit(char *z, const char *delim, int32_t *num) { char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { for (int32_t i = 0; i < len; ++i) { - // skip the needle in quote, jump to the end of quoted string if (skipquote && (haystack[i] == '\'' || haystack[i] == '"')) { char quote = haystack[i++]; - while(i < len && haystack[i++] != quote); + while (i < len && haystack[i++] != quote) + ; if (i >= len) { return NULL; } @@ -186,9 +186,9 @@ char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { return NULL; } -char* strtolower(char *dst, const char *src) { +char *strtolower(char *dst, const char *src) { int32_t esc = 0; - char quote = 0, *p = dst, c; + char quote = 0, *p = dst, c; assert(dst != NULL); @@ -213,9 +213,9 @@ char* strtolower(char *dst, const char *src) { return dst; } -char* strntolower(char *dst, const char *src, int32_t n) { +char *strntolower(char *dst, const char *src, int32_t n) { int32_t esc = 0; - char quote = 0, *p = dst, c; + char quote = 0, *p = dst, c; assert(dst != NULL); if (n == 0) { @@ -243,7 +243,7 @@ char* strntolower(char *dst, const char *src, int32_t n) { return dst; } -char* strntolower_s(char *dst, const char *src, int32_t n) { +char *strntolower_s(char *dst, const char *src, int32_t n) { char *p = dst, c; assert(dst != NULL); @@ -265,7 +265,7 @@ char* strntolower_s(char *dst, const char *src, int32_t n) { char *paGetToken(char *string, char **token, int32_t *tokenLen) { char quote = 0; - + while (*string != 0) { if (*string == ' ' || *string == '\t') { ++string; @@ -346,8 +346,8 @@ char *strbetween(char *string, char *begin, char *end) { char *result = NULL; char *_begin = strstr(string, begin); if (_begin != NULL) { - char *_end = strstr(_begin + strlen(begin), end); - int32_t size = (int32_t)(_end - _begin); + char *_end = strstr(_begin + strlen(begin), end); + int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { result = (char *)calloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); @@ -401,11 +401,12 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { } char *taosIpStr(uint32_t ipInt) { - static char ipStrArray[3][30]; + static char ipStrArray[3][30]; static int32_t ipStrIndex = 0; char *ipStr = ipStrArray[(ipStrIndex++) % 3]; - //sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); + // sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt + // >> 24)); sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); return ipStr; } diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index 8409637e80227b7a5c4e28a209cc71c659ce2e41..c70fdc87a6363b1a9fd08bddef3e1b3d0439162e 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -14,9 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "os.h" +#include "tversion.h" #include "taoserror.h" -#include "tdef.h" int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { if (vstr == NULL) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 2843f4e801acdde8a1b8818897efbb096ce32891..1657a85ee8c85ffc70e088ba307cdb37d4873417 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -71,8 +71,8 @@ static void *tQWorkerThreadFp(SQWorker *worker) { SQWorkerPool *pool = worker->pool; FItem fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -151,8 +151,8 @@ static void *tFWorkerThreadFp(SQWorker *worker) { SQWorkerPool *pool = worker->pool; FItem fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -240,8 +240,8 @@ static void *tWWorkerThreadFp(SWWorker *worker) { SWWorkerPool *pool = worker->pool; FItems fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t numOfMsgs = 0; int32_t qtype = 0; diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 800e261dcb308a09c45a0c8802b227446a94b7d0..712fb2d09a240037e99fd144a5030bb670ef606a 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -62,9 +62,9 @@ TEST_F(CfgTest, 02_Basic) { EXPECT_EQ(cfgGetSize(pConfig), 6); - int32_t size = 0; - SConfigItem *pItem = cfgIterate(pConfig, NULL); - while (pItem != NULL) { + int32_t size = taosArrayGetSize(pConfig->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = (SConfigItem *)taosArrayGet(pConfig->array, i); switch (pItem->dtype) { case CFG_DTYPE_BOOL: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); @@ -88,14 +88,10 @@ TEST_F(CfgTest, 02_Basic) { printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); break; } - size++; - pItem = cfgIterate(pConfig, pItem); } - cfgCancelIterate(pConfig, pItem); - EXPECT_EQ(cfgGetSize(pConfig), 6); - pItem = cfgGetItem(pConfig, "test_bool"); + SConfigItem *pItem = cfgGetItem(pConfig, "test_bool"); EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); EXPECT_STREQ(pItem->name, "test_bool"); diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 1b13c102a5a14103f51ceaf337d346723fd6f6f0..5505a6207f676d7f34d3476ca83cb33d24d76d9d 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -2,7 +2,7 @@ #include "gtest/gtest.h" -#include "encode.h" +#include "tencode.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshift-count-overflow" @@ -12,41 +12,41 @@ #define BUF_SIZE 64 td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN}; -static int encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } -static int encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } -static int encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); } -static int encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); } -static int encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); } -static int encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); } -static int encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); } -static int encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); } - -static int decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } -static int decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } -static int decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); } -static int decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); } -static int decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); } -static int decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); } -static int decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); } -static int decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); } - -static int encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } -static int encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } -static int encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); } -static int encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); } -static int encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); } -static int encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); } -static int encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); } -static int encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); } - -static int decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } -static int decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } -static int decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); } -static int decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); } -static int decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); } -static int decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); } -static int decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); } -static int decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); } +static int32_t encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int32_t encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); } +static int32_t encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); } +static int32_t encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); } + +static int32_t decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int32_t decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); } +static int32_t decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); } +static int32_t decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); } + +static int32_t encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); } + +static int32_t decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); } template static void simple_encode_decode_func(bool var_len) { @@ -211,7 +211,7 @@ typedef struct { char * A_c; } SStructA_v1; -static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { +static int32_t tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI32(pCoder, pSAV1->A_a) < 0) return -1; @@ -222,7 +222,7 @@ static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { return 0; } -static int tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { +static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32(pCoder, &pSAV1->A_a) < 0) return -1; @@ -246,7 +246,7 @@ typedef struct { int16_t A_e; } SStructA_v2; -static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { +static int32_t tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI32(pCoder, pSAV2->A_a) < 0) return -1; @@ -261,7 +261,7 @@ static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { return 0; } -static int tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { +static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32(pCoder, &pSAV2->A_a) < 0) return -1; @@ -291,7 +291,7 @@ typedef struct { int8_t v_b; } SFinalReq_v1; -static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { +static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { if (tStartEncode(pCoder) < 0) return -1; if (tSStructA_v1_encode(pCoder, ps1->pA) < 0) return -1; @@ -302,7 +302,7 @@ static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { return 0; } -static int tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { +static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder); @@ -322,7 +322,7 @@ typedef struct { int16_t v_c; } SFinalReq_v2; -static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { +static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { if (tStartEncode(pCoder) < 0) return -1; if (tSStructA_v2_encode(pCoder, ps2->pA) < 0) return -1; @@ -336,7 +336,7 @@ static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { return 0; } -static int tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { +static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder); diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 7a4e8be5b7f908fdccfe524c2ec2729a6090ec90..cf11d6b5bf461a19ce4a6ee231ee92d0c5fbdb8b 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -1,6 +1,6 @@ #include "gtest/gtest.h" -#include "freelist.h" +#include "tfreelist.h" TEST(TD_UTIL_FREELIST_TEST, simple_test) { SFreeList fl; diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 8fa82162234dd60c9562a0ad5103f818b26260c9..4b9d8c5f515f397d81d299d554278c132c03d9b2 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -13,144 +13,144 @@ namespace { // simple test void simpleTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4096, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/"); int32_t pageId = 0; int32_t groupId = 0; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1); releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t = getBufPage(pResultBuf, pageId); + SFilePage* t = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t == pBufPage1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage4); - SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage5); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } void writeDownTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); *(int32_t*)(pBufPage->data) = nx; writePageId = pageId; releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseBufPage(pResultBuf, t4); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 5); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } void recyclePageTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseBufPage(pResultBuf, t4); - SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t5 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t5 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t5 == pBufPage5); ASSERT_TRUE(pageId == 5); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); *(int32_t*)(pBufPagex->data) = nx; writePageId = pageId; // update the data releaseBufPage(pResultBuf, pBufPagex); - SFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); + SFilePage* pBufPagex1 = static_cast(getBufPage(pResultBuf, 1)); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 6); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } } // namespace diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index a1413be1ce4ce6f67516fc09121115f30bbc56f0..f2ab468574e29d178cf15cf47c618b11cd869030 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -19,7 +19,7 @@ #include "shell.h" #include "shellCommand.h" -#include "tkey.h" +#include "tbase64.h" #include "tscLog.h" diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index ce15212f8697f6c56fa3d07101576be2e099c00c..c0ab7ef461d3dcfa9df3b867153d255dd490c7ad 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -171,7 +171,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { */ // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index e43267f03217fd023959632010342d129e9c0624..5a989937d8cfd41050bca0a077947a6feda9514b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -517,7 +517,7 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } // FILE *fp = fopen(full_path.we_wordv[0], "w"); - TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); @@ -904,7 +904,7 @@ void read_history() { get_history_path(f_history); // FILE *f = fopen(f_history, "r"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { @@ -934,7 +934,7 @@ void write_history() { get_history_path(f_history); // FILE *f = fopen(f_history, "w"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); @@ -991,7 +991,7 @@ void source_file(TAOS *con, char *fptr) { */ // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index f304f35e3e60ecc41b83da6f5d134d22cbc8f0d0..57d6df00512014191d23fba41c4ee7c800556982 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -18,7 +18,7 @@ #include "tglobal.h" #include "shell.h" #include "shellCommand.h" -#include "tkey.h" +#include "tbase64.h" #include "tlog.h" #include "version.h"