diff --git a/2.0/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h index 9c738dad988e4be8d223711a83892e7623d1b27a..970b8263035420ff4bc67e1b192dc7f0f6ffc5f8 100644 --- a/2.0/src/query/inc/qExecutor.h +++ b/2.0/src/query/inc/qExecutor.h @@ -589,7 +589,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRunt SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createAllTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createFillOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c index ca656b81ff0c7c269c86bf2613ada1983afd22ae..490584c75a5e5fed0ea590189cb22f465f66e06f 100644 --- a/2.0/src/query/src/qExecutor.c +++ b/2.0/src/query/src/qExecutor.c @@ -2166,7 +2166,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf } case OP_TimeWindow: { pRuntimeEnv->proot = - createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + createIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); int32_t opType = pRuntimeEnv->proot->upstream[0]->operatorType; if (opType != OP_DummyInput && opType != OP_Join) { setTableScanFilterOperatorInfo(pRuntimeEnv->proot->upstream[0]->info, pRuntimeEnv->proot); @@ -6756,7 +6756,7 @@ SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); pInfo->pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 074014b0d7ba89cd7a712bbd88e13fe2952e20fa..21b8b661df9809d8631cf1e7e915eb78c500671e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -193,6 +193,7 @@ endif(${BUILD_WITH_TRAFT}) # LIBUV if(${BUILD_WITH_UV}) + add_compile_options(-Wno-sign-compare) add_subdirectory(libuv) endif(${BUILD_WITH_UV}) diff --git a/contrib/test/craft/raftMain.c b/contrib/test/craft/raftMain.c index bae083cf9451632a5d1e2b57b8546b60634674d1..b28adfaaca4426873d1c12c250ae1c55bc002938 100644 --- a/contrib/test/craft/raftMain.c +++ b/contrib/test/craft/raftMain.c @@ -10,7 +10,7 @@ #include #include #include "raftServer.h" -#include "common.h" +#include "tcommon.h" const char *exe_name; diff --git a/contrib/test/craft/raftServer.c b/contrib/test/craft/raftServer.c index ffec22e6463b0cafcfd986d9980e3466a7c9c20d..c4f347f6f5ae61d7d1d2579f5492447a022a99a6 100644 --- a/contrib/test/craft/raftServer.c +++ b/contrib/test/craft/raftServer.c @@ -1,5 +1,5 @@ #include -#include "common.h" +#include "tcommon.h" #include "raftServer.h" char *keys; diff --git a/contrib/test/craft/raftServer.h b/contrib/test/craft/raftServer.h index f4087cf1a9b8d664a56c58bf797e01f60cba032a..fb6f9d299617b85dafdabc70fabf7fece6d1a17f 100644 --- a/contrib/test/craft/raftServer.h +++ b/contrib/test/craft/raftServer.h @@ -11,7 +11,7 @@ extern "C" { #include #include "raft.h" #include "raft/uv.h" -#include "common.h" +#include "tcommon.h" // simulate a db store, just for test diff --git a/include/common/taosdef.h b/include/common/taosdef.h index e5a7a3563e1348b7450f84b936fcb95f7208dd0a..66cb0bb4ba5c48e6320455a94d5b6778c283fbe4 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -16,27 +16,19 @@ #ifndef _TD_COMMON_TAOS_DEF_H_ #define _TD_COMMON_TAOS_DEF_H_ +#include "taos.h" +#include "tdef.h" + #ifdef __cplusplus extern "C" { #endif -#include "taos.h" -#include "tdef.h" - typedef int64_t tb_uid_t; -#define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) -#define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) +#define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) #define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) -typedef enum { - TAOS_QTYPE_RPC = 1, - TAOS_QTYPE_FWD = 2, - TAOS_QTYPE_WAL = 3, - TAOS_QTYPE_CQ = 4, - TAOS_QTYPE_QUERY = 5 -} EQType; - typedef enum { TSDB_SUPER_TABLE = 1, // super table TSDB_CHILD_TABLE = 2, // table created from super table @@ -46,14 +38,6 @@ typedef enum { TSDB_TABLE_MAX = 6 } ETableType; -typedef enum { - TSDB_MOD_MNODE = 1, - TSDB_MOD_HTTP = 2, - TSDB_MOD_MONITOR = 3, - TSDB_MOD_MQTT = 4, - TSDB_MOD_MAX = 5 -} EModuleType; - typedef enum { TSDB_CHECK_ITEM_NETWORK, TSDB_CHECK_ITEM_MEM, diff --git a/include/common/common.h b/include/common/tcommon.h similarity index 98% rename from include/common/common.h rename to include/common/tcommon.h index 8fa2d03d6d3f79a220b6c10a38070a8a42026155..37d20cdb974b8f56b8f69b8ecba64815d882c7f7 100644 --- a/include/common/common.h +++ b/include/common/tcommon.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_COMMON_H -#define TDENGINE_COMMON_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_COMMON_DEF_H_ +#define _TD_COMMON_DEF_H_ #include "taosdef.h" #include "tarray.h" #include "tmsg.h" #include "tvariant.h" + +#ifdef __cplusplus +extern "C" { +#endif + // typedef struct STimeWindow { // TSKEY skey; // TSKEY ekey; @@ -214,7 +215,7 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); pRsp->pBlockData = NULL; - // for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + // for (int32_t i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); // tDeleteSSDataBlock(pDataBlock); //} @@ -279,4 +280,4 @@ typedef struct SSessionWindow { } #endif -#endif // TDENGINE_COMMON_H +#endif /*_TD_COMMON_DEF_H_*/ diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 4a73cf7dd21ceb733b04e8f5ba62bb83ebbc674c..89a0e6b4dc5adf1659fc2a582b64cc355b2c9c89 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + #ifndef _TD_COMMON_DATA_FORMAT_H_ #define _TD_COMMON_DATA_FORMAT_H_ @@ -65,40 +66,40 @@ typedef struct { uint16_t offset; // point offset in STpRow after the header part. } STColumn; -#define colType(col) ((col)->type) -#define colColId(col) ((col)->colId) -#define colBytes(col) ((col)->bytes) +#define colType(col) ((col)->type) +#define colColId(col) ((col)->colId) +#define colBytes(col) ((col)->bytes) #define colOffset(col) ((col)->offset) -#define colSetType(col, t) (colType(col) = (t)) +#define colSetType(col, t) (colType(col) = (t)) #define colSetColId(col, id) (colColId(col) = (id)) -#define colSetBytes(col, b) (colBytes(col) = (b)) +#define colSetBytes(col, b) (colBytes(col) = (b)) #define colSetOffset(col, o) (colOffset(col) = (o)) // ----------------- TSDB SCHEMA DEFINITION typedef struct { - int version; // version - int numOfCols; // Number of columns appended - int tlen; // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + + int32_t version; // version + int32_t numOfCols; // Number of columns appended + int32_t tlen; // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + // (bytes)) uint16_t flen; // First part length in a STpRow after the header part uint16_t vlen; // pure value part length, excluded the overhead (bytes only) STColumn columns[]; } STSchema; -#define schemaNCols(s) ((s)->numOfCols) -#define schemaVersion(s) ((s)->version) -#define schemaTLen(s) ((s)->tlen) -#define schemaFLen(s) ((s)->flen) -#define schemaVLen(s) ((s)->vlen) +#define schemaNCols(s) ((s)->numOfCols) +#define schemaVersion(s) ((s)->version) +#define schemaTLen(s) ((s)->tlen) +#define schemaFLen(s) ((s)->flen) +#define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(const STSchema *pSchema); -int tdEncodeSchema(void **buf, STSchema *pSchema); -void * tdDecodeSchema(void *buf, STSchema **pRSchema); +int32_t tdEncodeSchema(void **buf, STSchema *pSchema); +void *tdDecodeSchema(void *buf, STSchema **pRSchema); -static FORCE_INLINE int comparColId(const void *key1, const void *key2) { +static FORCE_INLINE int32_t comparColId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((STColumn *)key2)->colId) { @@ -116,26 +117,26 @@ static FORCE_INLINE STColumn *tdGetColOfID(STSchema *pSchema, int16_t colId) { // ----------------- SCHEMA BUILDER DEFINITION typedef struct { - int tCols; - int nCols; - int tlen; + int32_t tCols; + int32_t nCols; + int32_t tlen; uint16_t flen; uint16_t vlen; - int version; + int32_t version; STColumn *columns; } STSchemaBuilder; -#define TD_VTYPE_BITS 2 // val type +#define TD_VTYPE_BITS 2 // val type #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 -#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder +#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder #define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) #define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) -int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); +int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); -int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); +int32_t tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); // ----------------- Semantic timestamp key definition @@ -143,17 +144,17 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); typedef uint64_t TKEY; -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID #define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_DELETE_FLAG (((TKEY)1) << 62) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) +#define TKEY_DELETE_FLAG (((TKEY)1) << 62) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) +#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) #define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) -#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) -#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) +#define tdGetTKEY(key) (((TKEY)TABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) #define MAX_TS_KEY ((TSKEY)0x3fffffffffffffff) @@ -165,15 +166,15 @@ typedef uint64_t TKEY; // typedef uint64_t TKEY; #define TKEY TSKEY -#define TKEY_INVALID UINT64_MAX -#define TKEY_NULL TKEY_INVALID +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID #define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) -#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG)) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) -#define TKEY_IS_DELETED(tkey) (false) +#define TKEY_IS_DELETED(tkey) (false) -#define tdGetTKEY(key) (key) +#define tdGetTKEY(key) (key) #define tdGetKey(tskey) (tskey) #define MIN_TS_KEY ((TSKEY)0x8000000000000001) @@ -194,7 +195,7 @@ static FORCE_INLINE TKEY keyToTkey(TSKEY key) { return tdGetTKEY(lkey); } -static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { +static FORCE_INLINE int32_t tkeyComparFn(const void *tkey1, const void *tkey2) { TSKEY key1 = tdGetKey(*(TKEY *)tkey1); TSKEY key2 = tdGetKey(*(TKEY *)tkey2); @@ -225,17 +226,17 @@ typedef void *SDataRow; #define TD_DATA_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 -#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) -#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) -#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) -#define dataRowKey(r) tdGetKey(dataRowTKey(r)) -#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) -#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) -#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) +#define dataRowLen(r) (*(TDRowLenT *)(r)) // 0~65535 +#define dataRowEnd(r) POINTER_SHIFT(r, dataRowLen(r)) +#define dataRowVersion(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(int16_t))) +#define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) +#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) +#define dataRowKey(r) tdGetKey(dataRowTKey(r)) +#define dataRowSetLen(r, l) (dataRowLen(r) = (l)) +#define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) +#define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) -#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) +#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); @@ -243,7 +244,7 @@ void tdInitDataRow(SDataRow row, STSchema *pSchema); SDataRow tdDataRowDup(SDataRow row); // offset here not include dataRow header length -static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, +static FORCE_INLINE int32_t tdAppendDataColVal(SDataRow row, const void *value, bool isCopyVarData, int8_t type, int32_t offset) { assert(value != NULL); int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; @@ -268,7 +269,7 @@ static FORCE_INLINE int tdAppendDataColVal(SDataRow row, const void *value, bool } // offset here not include dataRow header length -static FORCE_INLINE int tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { +static FORCE_INLINE int32_t tdAppendColVal(SDataRow row, const void *value, int8_t type, int32_t offset) { return tdAppendDataColVal(row, value, true, type, offset); } @@ -281,25 +282,25 @@ static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t o } } -static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void *tdGetPtrToCol(SDataRow row, STSchema *pSchema, int32_t idx) { return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset); } -static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void *tdGetColOfRowBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; return tdGetRowDataOfCol(row, type, offset); } -static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE bool tdIsColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; return isNull(tdGetRowDataOfCol(row, type, offset), type); } -static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int idx) { +static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSchema, int32_t idx) { int16_t offset = TD_DATA_ROW_HEAD_SIZE + pSchema->columns[idx].offset; int8_t type = pSchema->columns[idx].type; int16_t bytes = pSchema->columns[idx].bytes; @@ -307,8 +308,8 @@ static FORCE_INLINE void tdSetColOfRowNullBySchema(SDataRow row, STSchema *pSche setNull(tdGetRowDataOfCol(row, type, offset), type, bytes); } -static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int dstIdx, SDataRow src, - STSchema *pSrcSchema, int srcIdx) { +static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSchema, int32_t dstIdx, SDataRow src, + STSchema *pSrcSchema, int32_t srcIdx) { int8_t type = pDstSchema->columns[dstIdx].type; assert(type == pSrcSchema->columns[srcIdx].type); void *pData = tdGetPtrToCol(dst, pDstSchema, dstIdx); @@ -364,13 +365,13 @@ typedef struct SDataCol { uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM uint8_t reserve : 7; int16_t colId; // column ID - int bytes; // column data bytes defined - int offset; // data offset in a SDataRow (including the header size) - int spaceSize; // Total space size for this column - int len; // column data length + int32_t bytes; // column data bytes defined + int32_t offset; // data offset in a SDataRow (including the header size) + int32_t spaceSize; // Total space size for this column + int32_t len; // column data length VarDataOffsetT *dataOff; // For binary and nchar data, the offset in the data column - void * pData; // Actual data pointer - void * pBitmap; // Bitmap pointer + void *pData; // Actual data pointer + void *pBitmap; // Bitmap pointer TSKEY ts; // only used in last NULL column } SDataCol; @@ -378,17 +379,17 @@ typedef struct SDataCol { #define isAllRowsNone(pCol) ((pCol)->len == 0) static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } -int tdAllocMemForCol(SDataCol *pCol, int maxPoints); +int32_t tdAllocMemForCol(SDataCol *pCol, int32_t maxPoints); -void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); -int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); -void *dataColSetOffset(SDataCol *pCol, int nEle); +void dataColInit(SDataCol *pDataCol, STColumn *pCol, int32_t maxPoints); +int32_t dataColAppendVal(SDataCol *pCol, const void *value, int32_t numOfRows, int32_t maxPoints); +void *dataColSetOffset(SDataCol *pCol, int32_t nEle); -bool isNEleNull(SDataCol *pCol, int nEle); +bool isNEleNull(SDataCol *pCol, int32_t nEle); #if 0 // Get the data pointer from a column-wised data -static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { +static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int32_t row) { if (isAllRowsNull(pCol)) { return getNullValue(pCol->type); } @@ -399,7 +400,7 @@ static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { } } -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { +static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { assert(rows > 0); if (IS_VAR_DATA_TYPE(pDataCol->type)) { @@ -412,15 +413,15 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { typedef struct { col_id_t maxCols; // max number of columns col_id_t numOfCols; // Total number of cols - int maxPoints; // max number of points - int numOfRows; - int sversion; // TODO: set sversion + int32_t maxPoints; // max number of points + int32_t numOfRows; + int32_t sversion; // TODO: set sversion SDataCol *cols; } SDataCols; -#define keyCol(pCols) (&((pCols)->cols[0])) // Key column +#define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] // the idx row of column-wised data -#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) +#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { if (pCols->numOfRows) { return dataColsTKeyAt(pCols, 0); @@ -429,7 +430,7 @@ static FORCE_INLINE TKEY dataColsTKeyFirst(SDataCols *pCols) { } } -static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int row) { +static FORCE_INLINE TSKEY dataColsKeyAtRow(SDataCols *pCols, int32_t row) { assert(row < pCols->numOfRows); return dataColsKeyAt(pCols, row); } @@ -458,12 +459,12 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { } } -SDataCols *tdNewDataCols(int maxCols, int maxRows); +SDataCols *tdNewDataCols(int32_t maxCols, int32_t maxRows); void tdResetDataCols(SDataCols *pCols); -int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); +int32_t tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); SDataCols *tdFreeDataCols(SDataCols *pCols); -int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull); +int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool forceSetNull); // ----------------- K-V data row structure /* |<-------------------------------------- len -------------------------------------------->| @@ -483,30 +484,30 @@ typedef struct { #define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) -#define kvRowLen(r) (*(TDRowLenT *)(r)) -#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) -#define kvRowSetLen(r, len) kvRowLen(r) = (len) -#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) -#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) -#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) +#define kvRowLen(r) (*(TDRowLenT *)(r)) +#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) +#define kvRowSetLen(r, len) kvRowLen(r) = (len) +#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) +#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) +#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) -#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) -#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) -#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) -#define kvRowKey(r) tdGetKey(kvRowTKey(r)) -#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) -#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) - -SKVRow tdKVRowDup(SKVRow row); -int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); -int tdEncodeKVRow(void **buf, SKVRow row); -void * tdDecodeKVRow(void *buf, SKVRow *row); -void tdSortKVRowByColIdx(SKVRow row); - -static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { +#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) +#define kvRowFree(r) tfree(r) +#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) +#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) +#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) +#define kvRowKey(r) tdGetKey(kvRowTKey(r)) +#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) +#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) + +SKVRow tdKVRowDup(SKVRow row); +int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); +int32_t tdEncodeKVRow(void **buf, SKVRow row); +void *tdDecodeKVRow(void *buf, SKVRow *row); +void tdSortKVRowByColIdx(SKVRow row); + +static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { @@ -528,7 +529,7 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { #if 0 // offset here not include kvRow header length -static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, +static FORCE_INLINE int32_t tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type, int32_t offset) { assert(value != NULL); int32_t toffset = offset + TD_KV_ROW_HEAD_SIZE; @@ -581,15 +582,15 @@ typedef struct { SColIdx *pColIdx; uint16_t alloc; uint16_t size; - void * buf; + void *buf; } SKVRowBuilder; -int tdInitKVRowBuilder(SKVRowBuilder *pBuilder); -void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); -void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); -SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); +int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); +void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); +void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); +SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); -static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { +static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, const void *value) { if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; SColIdx *pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); @@ -602,7 +603,7 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, pBuilder->nCols++; - int tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; + int32_t tlen = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; if (tlen > pBuilder->alloc - pBuilder->size) { while (tlen > pBuilder->alloc - pBuilder->size) { pBuilder->alloc *= 2; @@ -643,24 +644,24 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, typedef void *SMemRow; -#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) -#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) +#define TD_MEM_ROW_TYPE_SIZE sizeof(uint8_t) +#define TD_MEM_ROW_KV_VER_SIZE sizeof(int16_t) #define TD_MEM_ROW_KV_TYPE_VER_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE) -#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) -#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) +#define TD_MEM_ROW_DATA_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_DATA_ROW_HEAD_SIZE) +#define TD_MEM_ROW_KV_HEAD_SIZE (TD_MEM_ROW_TYPE_SIZE + TD_MEM_ROW_KV_VER_SIZE + TD_KV_ROW_HEAD_SIZE) -#define SMEM_ROW_DATA 0x0U // SDataRow -#define SMEM_ROW_KV 0x01U // SKVRow +#define SMEM_ROW_DATA 0x0U // SDataRow +#define SMEM_ROW_KV 0x01U // SKVRow #define KVRatioConvert (0.9f) #define memRowType(r) ((*(uint8_t *)(r)) & 0x01) -#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory -#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) -#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) -#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) -#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) +#define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t)) // set the total byte in case of dirty memory +#define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01)) +#define isDataRow(r) (SMEM_ROW_DATA == memRowType(r)) +#define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01)) +#define isKvRow(r) (SMEM_ROW_KV == memRowType(r)) #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE) // section after flag @@ -668,14 +669,14 @@ typedef void *SMemRow; POINTER_SHIFT(r, TD_MEM_ROW_KV_TYPE_VER_SIZE) // section after flag + sversion as to reuse SKVRow #define memRowDataLen(r) (*(TDRowLenT *)memRowDataBody(r)) // 0~65535 -#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 +#define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r)) // 0~65535 #define memRowDataTLen(r) \ ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE)) // using uint32_t/int32_t to store the TLen #define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE)) -#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) +#define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r)) #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r)) // using uint32_t/int32_t to store the TLen static FORCE_INLINE char *memRowEnd(SMemRow row) { @@ -686,14 +687,14 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { } } -#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) -#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) -#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version +#define memRowDataVersion(r) dataRowVersion(memRowDataBody(r)) +#define memRowKvVersion(r) (*(int16_t *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)) +#define memRowVersion(r) (isDataRow(r) ? memRowDataVersion(r) : memRowKvVersion(r)) // schema version #define memRowSetKvVersion(r, v) (memRowKvVersion(r) = (v)) -#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) +#define memRowTuple(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowValues(memRowKvBody(r))) #define memRowTKey(r) (isDataRow(r) ? dataRowTKey(memRowDataBody(r)) : kvRowTKey(memRowKvBody(r))) -#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) +#define memRowKey(r) (isDataRow(r) ? dataRowKey(memRowDataBody(r)) : kvRowKey(memRowKvBody(r))) #define memRowKeys(r) (isDataRow(r) ? dataRowTuple(memRowDataBody(r)) : kvRowKeys(memRowKvBody(r))) #define memRowSetTKey(r, k) \ do { \ @@ -704,11 +705,11 @@ static FORCE_INLINE char *memRowEnd(SMemRow row) { } \ } while (0) -#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) -#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) -#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) +#define memRowSetLen(r, l) (isDataRow(r) ? memRowDataLen(r) = (l) : memRowKvLen(r) = (l)) +#define memRowSetVersion(r, v) (isDataRow(r) ? dataRowSetVersion(memRowDataBody(r), v) : memRowSetKvVersion(r, v)) +#define memRowCpy(dst, r) memcpy((dst), (r), memRowTLen(r)) #define memRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_MEM_ROW_DATA_HEAD_SIZE) -#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) +#define memRowDeleted(r) TKEY_IS_DELETED(memRowTKey(r)) SMemRow tdMemRowDup(SMemRow row); void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull); @@ -736,7 +737,7 @@ static FORCE_INLINE void *tdGetMemRowDataOfColEx(void *row, int16_t colId, int8_ } } -static FORCE_INLINE int tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, +static FORCE_INLINE int32_t tdAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId, int8_t type, int32_t offset) { if (isDataRow(row)) { tdAppendDataColVal(memRowDataBody(row), value, isCopyVarData, type, offset); diff --git a/include/common/tep.h b/include/common/tep.h index 584b8a5a71fb7833ea78c9991933c042099b237f..602df8ca53c7eab4e7881598286b27a934a675dc 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -1,14 +1,28 @@ -#ifndef TDENGINE_TEP_H -#define TDENGINE_TEP_H +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_EP_H_ +#define _TD_COMMON_EP_H_ + +#include "tcommon.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tmsg.h" -#include "common.h" - typedef struct SCorEpSet { int32_t version; SEpSet epSet; @@ -17,19 +31,16 @@ typedef struct SCorEpSet { typedef struct SBlockOrderInfo { int32_t order; int32_t colIndex; - SColumnInfoData *pColData; -// int32_t type; -// int32_t bytes; -// bool hasNull; + SColumnInfoData* pColData; } SBlockOrderInfo; -int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); -void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); +int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp); +void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port); -bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); +bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2); -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); -SEpSet getEpSet_s(SCorEpSet *pEpSet); +void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet); +SEpSet getEpSet_s(SCorEpSet* pEpSet); #define NBIT (3u) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) @@ -41,7 +52,8 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) -static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { +static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, + SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { return false; } @@ -67,22 +79,27 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u } } -#define colDataGet(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \ - : (p1_)->pData + ((r_) * (p1_)->info.bytes)); +#define BitmapLen(_n) (((_n) + ((1<> NBIT) + + +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? ((p1_)->pData + (p1_)->varmeta.offset[(r_)]) \ + : ((p1_)->pData + ((r_) * (p1_)->info.bytes))) int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); -int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); -void colDataTrim(SColumnInfoData* pColumnInfoData); +int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); +void colDataTrim(SColumnInfoData* pColumnInfoData); size_t colDataGetNumOfCols(const SSDataBlock* pBlock); size_t colDataGetNumOfRows(const SSDataBlock* pBlock); -int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize); SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); @@ -93,17 +110,20 @@ size_t blockDataGetRowSize(const SSDataBlock* pBlock); double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); -size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize); +SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); -void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); -void *blockDataDestroy(SSDataBlock *pBlock); +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); +SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock); +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); +void* blockDataDestroy(SSDataBlock* pBlock); #ifdef __cplusplus } #endif -#endif // TDENGINE_TEP_H +#endif /*_TD_COMMON_EP_H_*/ diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 4bc8c9fb86c03182af16b6ea0f38248982243a04..e09f6d11bdea1f3dbf5a08ab37d3086c9c1d2ed4 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -16,57 +16,52 @@ #ifndef _TD_COMMON_GLOBAL_H_ #define _TD_COMMON_GLOBAL_H_ +#include "tarray.h" +#include "tdef.h" + #ifdef __cplusplus extern "C" { #endif -#include "tdef.h" -#include "tcfg.h" - // cluster extern char tsFirst[]; extern char tsSecond[]; extern char tsLocalFqdn[]; extern char tsLocalEp[]; extern uint16_t tsServerPort; +extern int32_t tsVersion; extern int32_t tsStatusInterval; -extern int8_t tsEnableTelemetryReporting; -extern int32_t tsNumOfSupportVnodes; +extern bool tsEnableTelemetryReporting; // common -extern int tsRpcTimer; -extern int tsRpcMaxTime; -extern int tsRpcForceTcp; // all commands go to tcp protocol if this is enabled -extern int32_t tsMaxConnections; -extern int32_t tsMaxShellConns; -extern int32_t tsShellActivityTimer; -extern uint32_t tsMaxTmrCtrl; -extern float tsNumOfThreadsPerCore; -extern int32_t tsNumOfCommitThreads; -extern float tsRatioOfQueryCores; -extern int8_t tsDaylight; -extern int8_t tsEnableCoreFile; -extern int32_t tsCompressMsgSize; -extern int32_t tsCompressColData; -extern int32_t tsMaxNumOfDistinctResults; -extern char tsTempDir[]; -extern int tsCompatibleModel; // 2.0 compatible model -extern int8_t tsEnableSlaveQuery; -extern int8_t tsEnableAdjustMaster; -extern int8_t tsPrintAuth; -extern int64_t tsTickPerDay[3]; +extern int32_t tsRpcTimer; +extern int32_t tsRpcMaxTime; +extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled +extern int32_t tsMaxConnections; +extern int32_t tsMaxShellConns; +extern int32_t tsShellActivityTimer; +extern int32_t tsMaxTmrCtrl; +extern float tsNumOfThreadsPerCore; +extern int32_t tsNumOfCommitThreads; +extern float tsRatioOfQueryCores; +extern int32_t tsCompressMsgSize; +extern int32_t tsCompressColData; +extern int32_t tsMaxNumOfDistinctResults; +extern int32_t tsCompatibleModel; +extern bool tsEnableSlaveQuery; +extern bool tsPrintAuth; +extern int64_t tsTickPerDay[3]; // query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int64_t tsQueryBufferSizeBytes; // maximum allowed usage buffer size in byte for each data node -extern int32_t tsRetrieveBlockingModel; // retrieve threads will be blocked -extern int8_t tsKeepOriginalColumnName; -extern int8_t tsDeadLockKillQuery; +extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked +extern bool tsKeepOriginalColumnName; +extern bool tsDeadLockKillQuery; // client extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; -extern int8_t tsTscEnableRecordSql; extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; @@ -77,19 +72,6 @@ extern float tsStreamComputDelayRatio; // the delayed computing ration of the extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; -// system info -extern float tsTotalLogDirGB; -extern float tsTotalTmpDirGB; -extern float tsTotalDataDirGB; -extern float tsAvailLogDirGB; -extern float tsAvailTmpDirectorySpace; -extern float tsAvailDataDirGB; -extern float tsUsedDataDirGB; -extern float tsMinimalLogDirGB; -extern float tsReservedTmpDirectorySpace; -extern float tsMinimalDataDirGB; -extern uint32_t tsVersion; - // build info extern char version[]; extern char compatible_version[]; @@ -105,18 +87,19 @@ extern uint32_t tsMaxRange; extern uint32_t tsCurRange; extern char tsCompressor[]; +// tfs extern int32_t tsDiskCfgNum; extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -void taosInitGlobalCfg(); -int32_t taosCheckAndPrintCfg(); -int32_t taosCfgDynamicOptions(char *msg); -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId); -void taosAddDataDir(int index, char *v1, int level, int primary); -void taosReadDataDirCfg(char *v1, char *v2, char *v3); -void taosPrintDataDirCfg(); +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, SArray *pArgs, bool tsc); +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc); +void taosCleanupCfg(); +void taosCfgDynamicOptions(const char *option, const char *value); + +struct SConfig *taosGetCfg(); #ifdef __cplusplus } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5d989421f6e3df2be213d9d1346a8624083d8da4..ebd4563f8c09bbd96d9adaa656f93b0fee29e7a7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -16,19 +16,19 @@ #ifndef _TD_COMMON_TAOS_MSG_H_ #define _TD_COMMON_TAOS_MSG_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include "encode.h" #include "taosdef.h" #include "taoserror.h" #include "tarray.h" #include "tcoding.h" +#include "tencode.h" #include "thash.h" #include "tlist.h" #include "trow.h" +#ifdef __cplusplus +extern "C" { +#endif + /* ------------------------ MESSAGE DEFINITIONS ------------------------ */ #define TD_MSG_NUMBER_ #undef TD_MSG_DICT_ @@ -48,8 +48,8 @@ extern "C" { #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -extern char* tMsgInfo[]; -extern int tMsgDict[]; +extern char* tMsgInfo[]; +extern int32_t tMsgDict[]; #define TMSG_SEG_CODE(TYPE) (((TYPE)&0xff00) >> 8) #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) @@ -655,10 +655,10 @@ int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp typedef struct { int32_t statusInterval; - int64_t checkTime; // 1970-01-01 00:00:00.000 - char timezone[TSDB_TIMEZONE_LEN]; // tsTimezone - char locale[TSDB_LOCALE_LEN]; // tsLocale - char charset[TSDB_LOCALE_LEN]; // tsCharset + int64_t checkTime; // 1970-01-01 00:00:00.000 + char timezone[TD_TIMEZONE_LEN]; // tsTimezone + char locale[TD_LOCALE_LEN]; // tsLocale + char charset[TD_LOCALE_LEN]; // tsCharset } SClusterCfg; typedef struct { @@ -1117,13 +1117,13 @@ typedef struct { SArray* topicNames; // SArray } SCMSubscribeReq; -static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->consumerGroup); - for (int i = 0; i < pReq->topicNum; i++) { + for (int32_t i = 0; i < pReq->topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; @@ -1134,7 +1134,7 @@ static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeString(buf, &pReq->consumerGroup); pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); - for (int i = 0; i < pReq->topicNum; i++) { + for (int32_t i = 0; i < pReq->topicNum; i++) { char* name; buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); @@ -1153,10 +1153,10 @@ typedef struct { SMqSubTopic topics[]; } SCMSubscribeRsp; -static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pRsp->topicNum); - for (int i = 0; i < pRsp->topicNum; i++) { + for (int32_t i = 0; i < pRsp->topicNum; i++) { tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId); tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId); tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet); @@ -1166,7 +1166,7 @@ static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribe static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { buf = taosDecodeFixedI32(buf, &pRsp->topicNum); - for (int i = 0; i < pRsp->topicNum; i++) { + for (int32_t i = 0; i < pRsp->topicNum; i++) { buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId); buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId); buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet); @@ -1184,8 +1184,8 @@ typedef struct { char* physicalPlan; } SMVSubscribeReq; -static FORCE_INLINE int tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { - int tlen = 0; +static FORCE_INLINE int32_t tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) { + int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pReq->topicId); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId); @@ -1395,8 +1395,8 @@ typedef struct SMqHbRsp { SEpSet epSet; } SMqHbRsp; -static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pRsp->status); tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); tlen += taosEncodeFixedI8(buf, pRsp->epChanged); @@ -1417,8 +1417,8 @@ typedef struct SMqHbOneTopicBatchRsp { SArray* rsps; // SArray } SMqHbOneTopicBatchRsp; -static FORCE_INLINE int taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) { + int32_t tlen = 0; tlen += taosEncodeString(buf, pBatchRsp->topicName); int32_t sz = taosArrayGetSize(pBatchRsp->rsps); tlen += taosEncodeFixedI32(buf, sz); @@ -1447,8 +1447,8 @@ typedef struct SMqHbBatchRsp { SArray* batchRsps; // SArray } SMqHbBatchRsp; -static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { + int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId); int32_t sz; tlen += taosEncodeFixedI32(buf, sz); @@ -1589,8 +1589,8 @@ typedef struct SMqHbVgInfo { int32_t vgId; } SMqHbVgInfo; -static FORCE_INLINE int taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); return tlen; } @@ -1607,8 +1607,8 @@ typedef struct SMqHbTopicInfo { SArray* pVgInfo; } SMqHbTopicInfo; -static FORCE_INLINE int taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); tlen += taosEncodeString(buf, pTopicInfo->name); @@ -1643,14 +1643,14 @@ typedef struct SMqHbMsg { SArray* pTopics; // SArray } SMqHbMsg; -static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { + int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pMsg->status); tlen += taosEncodeFixedI32(buf, pMsg->epoch); tlen += taosEncodeFixedI64(buf, pMsg->consumerId); int32_t sz = taosArrayGetSize(pMsg->pTopics); tlen += taosEncodeFixedI32(buf, sz); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); } @@ -1664,7 +1664,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { SMqHbTopicInfo topicInfo; buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); taosArrayPush(pMsg->pTopics, &topicInfo); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 18718b0c52da5701026aa4149b2a1fbe03f47aca..cae186ba16ed7af71d888eee367f13b067b712d5 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -47,7 +47,7 @@ enum { #define TD_NEW_MSG_SEG(TYPE) TYPE##_NUM, #define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) -int tMsgDict[] = { +int32_t tMsgDict[] = { #elif defined(TD_MSG_SEG_CODE_) @@ -127,7 +127,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STB_META, "mnode-stb-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TABLE_META, "mnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) @@ -135,8 +135,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "mnode-trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "mnode-kill-trans", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "mnode-telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) @@ -144,7 +145,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) diff --git a/include/common/tname.h b/include/common/tname.h index 47028fbce18945f1b33f57d5d5ccaba2a730103a..6de38a68ee9ac782eb52e02f097d24633331ea13 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -13,26 +13,25 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TNAME_H -#define TDENGINE_TNAME_H +#ifndef _TD_COMMON_NAME_H_ +#define _TD_COMMON_NAME_H_ + +#include "tdef.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 -#include "tdef.h" -#include "tmsg.h" - -#define TSDB_DB_NAME_T 1 -#define TSDB_TABLE_NAME_T 2 - -#define T_NAME_ACCT 0x1u -#define T_NAME_DB 0x2u -#define T_NAME_TABLE 0x4u +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u typedef struct SName { - uint8_t type; //db_name_t, table_name_t + uint8_t type; // db_name_t, table_name_t int32_t acctId; char dbname[TSDB_DB_NAME_LEN]; char tname[TSDB_TABLE_NAME_LEN]; @@ -68,5 +67,4 @@ SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* nam } #endif - -#endif // TDENGINE_TNAME_H +#endif /*_TD_COMMON_NAME_H_*/ diff --git a/include/common/trequest.h b/include/common/trequest.h index 6932122bc92553255f6989746cbcb091b6f201c6..05abadcb4431f550f36b31c733f278f852e2221b 100644 --- a/include/common/trequest.h +++ b/include/common/trequest.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TREQUEST_H_ -#define _TD_TREQUEST_H_ +#ifndef _TD_COMMON_REQUEST_H_ +#define _TD_COMMON_REQUEST_H_ #ifdef __cplusplus extern "C" { @@ -36,7 +36,7 @@ void tdClearRBIter(SReqBatchIter *pIter); /* ------------------------ TYPES DEFINITION ------------------------ */ struct SReqBatchIter { - int iReq; + int32_t iReq; SReqBatch *pReqBatch; }; @@ -44,4 +44,4 @@ struct SReqBatchIter { } #endif -#endif /*_TD_TREQUEST_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_REQUEST_H_*/ \ No newline at end of file diff --git a/include/common/trow.h b/include/common/trow.h index fad74a359dad164cc18ad7bb1f06b90b116ace24..49bc2f3515a3304e8421d3f6f4cdc33a901be430 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -18,11 +18,11 @@ #include "os.h" #include "talgo.h" +#include "taosdef.h" #include "taoserror.h" #include "tbuffer.h" -#include "tdef.h" -#include "taosdef.h" #include "tdataformat.h" +#include "tdef.h" #include "tschema.h" #include "ttypes.h" #include "tutil.h" @@ -47,21 +47,21 @@ extern "C" { #define TD_VTYPE_NONE 0x0U // none or unknown/undefined #define TD_VTYPE_NULL 0x01U // null val #define TD_VTYPE_NORM 0x02U // normal val: not none, not null -#define TD_VTYPE_MAX 0x03U // +#define TD_VTYPE_MAX 0x03U // #define TD_VTYPE_NONE_BYTE 0x0U #define TD_VTYPE_NULL_BYTE 0x55U #define TD_VTYPE_NORM_BYTE 0xAAU -#define TD_ROWS_ALL_NORM 0x01U +#define TD_ROWS_ALL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x0U -#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM +#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) -#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) -#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) +#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) +#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) -#define KvConvertRatio (0.9f) +#define KvConvertRatio (0.9f) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #ifdef TD_SUPPORT_BITMAP @@ -98,7 +98,7 @@ typedef void *SRow; typedef struct { TDRowValT valType; - void * val; + void *val; } SCellVal; typedef struct { @@ -157,41 +157,41 @@ typedef struct { int16_t nBitmaps; int16_t nBoundBitmaps; int32_t offset; - void * pBitmap; - void * pOffset; + void *pBitmap; + void *pOffset; int32_t extendedRowSize; } SRowBuilder; -#define TD_ROW_HEAD_LEN (sizeof(STSRow)) +#define TD_ROW_HEAD_LEN (sizeof(STSRow)) #define TD_ROW_NCOLS_LEN (sizeof(col_id_t)) -#define TD_ROW_TYPE(r) ((r)->type) -#define TD_ROW_DELETE(r) ((r)->del) -#define TD_ROW_ENDIAN(r) ((r)->endian) -#define TD_ROW_SVER(r) ((r)->sver) -#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow -#define TD_ROW_DATA(r) ((r)->data) -#define TD_ROW_LEN(r) ((r)->len) -#define TD_ROW_KEY(r) ((r)->ts) +#define TD_ROW_TYPE(r) ((r)->type) +#define TD_ROW_DELETE(r) ((r)->del) +#define TD_ROW_ENDIAN(r) ((r)->endian) +#define TD_ROW_SVER(r) ((r)->sver) +#define TD_ROW_NCOLS(r) ((r)->data) // only valid for SKvRow +#define TD_ROW_DATA(r) ((r)->data) +#define TD_ROW_LEN(r) ((r)->len) +#define TD_ROW_KEY(r) ((r)->ts) #define TD_ROW_KEY_ADDR(r) POINTER_SHIFT((r), 16) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and -// (int)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. +// (int32_t)ceil((double)nCols/TD_VTYPE_PARTS) should be added if TD_SUPPORT_BITMAP defined. #define TD_ROW_MAX_BYTES_FROM_SCHEMA(s) (schemaTLen(s) + TD_ROW_HEAD_LEN) -#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) -#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) -#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) -#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) +#define TD_ROW_SET_TYPE(r, t) (TD_ROW_TYPE(r) = (t)) +#define TD_ROW_SET_DELETE(r) (TD_ROW_DELETE(r) = 1) +#define TD_ROW_SET_SVER(r, v) (TD_ROW_SVER(r) = (v)) +#define TD_ROW_SET_LEN(r, l) (TD_ROW_LEN(r) = (l)) #define TD_ROW_SET_NCOLS(r, n) (*(col_id_t *)TD_ROW_NCOLS(r) = (n)) #define TD_ROW_IS_DELETED(r) (TD_ROW_DELETE(r) == 1) -#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) -#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) -#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) -#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) +#define TD_IS_TP_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_TP) +#define TD_IS_KV_ROW(r) (TD_ROW_TYPE(r) == TD_ROW_KV) +#define TD_IS_TP_ROW_T(t) ((t) == TD_ROW_TP) +#define TD_IS_KV_ROW_T(t) ((t) == TD_ROW_KV) -#define TD_BOOL_STR(b) ((b) ? "true" : "false") +#define TD_BOOL_STR(b) ((b) ? "true" : "false") #define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert)) #define TD_ROW_COL_IDX(r) POINTER_SHIFT(TD_ROW_DATA(r), sizeof(col_id_t)) @@ -217,7 +217,7 @@ static FORCE_INLINE void tdRowCopy(void *dst, STSRow *row) { memcpy(dst, row, static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType); static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType); -int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints); +int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset); static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -272,7 +272,7 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pDestByte = ((*pDestByte) & 0x3F) | (valType << 6); @@ -310,7 +310,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD } int16_t nBytes = colIdx / TD_VTYPE_PARTS; int16_t nOffset = colIdx & TD_VTYPE_OPTR; - char * pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); + char *pDestByte = (char *)POINTER_SHIFT(pBitmap, nBytes); switch (nOffset) { case 0: *pValType = (((*pDestByte) & 0xC0) >> 6); @@ -355,7 +355,6 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD * */ - /** * @brief * @@ -432,7 +431,6 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t pBuilder->rowType = TD_ROW_TP; } - } else { pBuilder->rowType = TD_ROW_TP; } @@ -618,7 +616,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa if (tdValIsNorm(valType, val, colType)) { // ts key stored in STSRow.ts SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN @@ -636,7 +634,7 @@ static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowVa // NULL/None value else { SKvRowIdx *pColIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(row), offset); - char * ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); + char *ptr = (char *)POINTER_SHIFT(row, TD_ROW_LEN(row)); pColIdx->colId = colId; pColIdx->offset = TD_ROW_LEN(row); // the offset include the TD_ROW_HEAD_LEN const void *nullVal = getNullValue(colType); @@ -731,7 +729,7 @@ static FORCE_INLINE int32_t tdGetTpRowValOfCol(SCellVal *output, STSRow *pRow, v return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int compareKvRowColId(const void *key1, const void *key2) { +static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { return 1; } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { @@ -772,8 +770,8 @@ static FORCE_INLINE int32_t tdGetKvRowValOfCol(SCellVal *output, STSRow *pRow, v typedef struct { STSchema *pSchema; - STSRow * pRow; - void * pBitmap; + STSRow *pRow; + void *pBitmap; uint32_t offset; col_id_t maxColId; col_id_t colIdx; // [PRIMARYKEY_TIMESTAMP_COL_ID, nSchemaCols], PRIMARYKEY_TIMESTAMP_COL_ID equals 1 @@ -793,8 +791,8 @@ static FORCE_INLINE void tdSTSRowIterInit(STSRowIter *pIter, STSchema *pSchema) pIter->maxColId = pSchema->columns[pSchema->numOfCols - 1].colId; } -static int tdCompareColId(const void *arg1, const void *arg2) { - int colId = *(int *)arg1; +static int32_t tdCompareColId(const void *arg1, const void *arg2) { + int32_t colId = *(int32_t *)arg1; STColumn *pCol = (STColumn *)arg2; if (colId < pCol->colId) { @@ -878,7 +876,7 @@ static FORCE_INLINE bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colTy // internal static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId, col_type_t colType, col_id_t *nIdx, SCellVal *pVal) { - STSRow * pRow = pIter->pRow; + STSRow *pRow = pIter->pRow; SKvRowIdx *pKvIdx = NULL; bool colFound = false; col_id_t kvNCols = tdRowGetNCols(pRow); @@ -959,11 +957,11 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col STSRow *mergeTwoRows(void *buffer, STSRow *row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2); // Get the data pointer from a column-wised data -static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int row) { +static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, int32_t row) { if (isAllRowsNone(pCol)) { pVal->valType = TD_VTYPE_NULL; #ifdef TD_SUPPORT_READ2 - pVal->val = (void*)getNullValue(pCol->type); + pVal->val = (void *)getNullValue(pCol->type); #else pVal->val = NULL; #endif @@ -981,7 +979,7 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } else { pVal->valType = TD_VTYPE_NULL; #ifdef TD_SUPPORT_READ2 - pVal->val = (void*)getNullValue(pCol->type); + pVal->val = (void *)getNullValue(pCol->type); #else pVal->val = NULL; #endif @@ -1000,7 +998,8 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t return true; } -static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { +static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, + SCellVal *pVal) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { tdRowSetVal(pVal, TD_VTYPE_NORM, TD_ROW_KEY_ADDR(pRow)); return true; @@ -1010,7 +1009,7 @@ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t o return true; } -static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { +static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int32_t rows) { ASSERT(rows > 0); int32_t result = 0; @@ -1068,7 +1067,7 @@ typedef struct { typedef struct { STSchema *pSchema; - STSRow * pRow; + STSRow *pRow; } STSRowReader; typedef struct { @@ -1079,15 +1078,15 @@ typedef struct { // STSRowBuilder #define trbInit(rt, allocator, endian, target, size) \ { .type = (rt), .bw = tbufInitWriter(allocator, endian), .pRow = (target) } -void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); -void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); -void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); -int trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); +void trbSetRowInfo(STSRowBuilder *pRB, bool del, uint16_t sver); +void trbSetRowVersion(STSRowBuilder *pRB, uint64_t ver); +void trbSetRowTS(STSRowBuilder *pRB, TSKEY ts); +int32_t trbWriteCol(STSRowBuilder *pRB, void *pData, col_id_t cid); // STSRowReader #define tRowReaderInit(schema, row) \ { .schema = (schema), .row = (row) } -int tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); +int32_t tRowReaderRead(STSRowReader *pRowReader, col_id_t cid, void *target, uint64_t size); // STSRowBatchIter #define tRowBatchIterInit(pRB) \ diff --git a/include/common/tschema.h b/include/common/tschema.h index cfa0eeace2a0096492d4bb61e32359f0fe9e5e80..7a270fc9d9d247b5d2ea4483b36969225d5a880e 100644 --- a/include/common/tschema.h +++ b/include/common/tschema.h @@ -70,7 +70,7 @@ typedef struct { { .size = (capacity), .pSchema = (target) } void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver); void tSchemaBuilderSetSMA(bool sma, SArray *smaArray); -int tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment); +int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment); #endif diff --git a/include/common/ttime.h b/include/common/ttime.h index e22e1d6c4ed9aeb406abce8468d3f92f4599c298..a4b2aa6673c4413225ff0bfdd5b58851f400d2b0 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -16,13 +16,13 @@ #ifndef _TD_COMMON_TIME_H_ #define _TD_COMMON_TIME_H_ +#include "taosdef.h" +#include "tmsg.h" + #ifdef __cplusplus extern "C" { #endif -#include "taosdef.h" -#include "tmsg.h" - #define TIME_IS_VAR_DURATION(_t) ((_t) == 'n' || (_t) == 'y' || (_t) == 'N' || (_t) == 'Y') /* @@ -35,7 +35,7 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) { return taosGetTimestampUs(); } else if (precision == TSDB_TIME_PRECISION_NANO) { return taosGetTimestampNs(); - }else { + } else { return taosGetTimestampMs(); } } @@ -56,4 +56,4 @@ int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrec } #endif -#endif /*_TD_COMMON_TIME_H_*/ +#endif /*_TD_COMMON_TIME_H_*/ diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 7501f61983ad36d943966d33b47138cda0f38068..f86cbff700c97bcaabad867ce05b705a85efcff5 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -13,284 +13,282 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTOKENDEF_H -#define TDENGINE_TTOKENDEF_H +#ifndef _TD_COMMON_TOKEN_H_ +#define _TD_COMMON_TOKEN_H_ -#define TK_ID 1 -#define TK_BOOL 2 -#define TK_INTEGER 3 -#define TK_FLOAT 4 -#define TK_STRING 5 -#define TK_TIMESTAMP 6 -#define TK_OR 7 -#define TK_AND 8 -#define TK_NOT 9 -#define TK_EQ 10 -#define TK_NE 11 -#define TK_ISNULL 12 -#define TK_NOTNULL 13 -#define TK_IS 14 -#define TK_LIKE 15 -#define TK_MATCH 16 -#define TK_NMATCH 17 -#define TK_GLOB 18 -#define TK_BETWEEN 19 -#define TK_IN 20 -#define TK_GT 21 -#define TK_GE 22 -#define TK_LT 23 -#define TK_LE 24 -#define TK_BITAND 25 -#define TK_BITOR 26 -#define TK_LSHIFT 27 -#define TK_RSHIFT 28 -#define TK_PLUS 29 -#define TK_MINUS 30 -#define TK_DIVIDE 31 -#define TK_TIMES 32 -#define TK_STAR 33 -#define TK_SLASH 34 -#define TK_REM 35 -#define TK_CONCAT 36 -#define TK_UMINUS 37 -#define TK_UPLUS 38 -#define TK_BITNOT 39 -#define TK_SHOW 40 -#define TK_DATABASES 41 -#define TK_TOPICS 42 -#define TK_FUNCTIONS 43 -#define TK_MNODES 44 -#define TK_DNODES 45 -#define TK_ACCOUNTS 46 -#define TK_USERS 47 -#define TK_MODULES 48 -#define TK_QUERIES 49 -#define TK_CONNECTIONS 50 -#define TK_STREAMS 51 -#define TK_VARIABLES 52 -#define TK_SCORES 53 -#define TK_GRANTS 54 -#define TK_VNODES 55 -#define TK_DOT 56 -#define TK_CREATE 57 -#define TK_TABLE 58 -#define TK_STABLE 59 -#define TK_DATABASE 60 -#define TK_TABLES 61 -#define TK_STABLES 62 -#define TK_VGROUPS 63 -#define TK_DROP 64 -#define TK_TOPIC 65 -#define TK_FUNCTION 66 -#define TK_DNODE 67 -#define TK_USER 68 -#define TK_ACCOUNT 69 -#define TK_USE 70 -#define TK_DESCRIBE 71 -#define TK_DESC 72 -#define TK_ALTER 73 -#define TK_PASS 74 -#define TK_PRIVILEGE 75 -#define TK_LOCAL 76 -#define TK_COMPACT 77 -#define TK_LP 78 -#define TK_RP 79 -#define TK_IF 80 -#define TK_EXISTS 81 -#define TK_PORT 82 -#define TK_IPTOKEN 83 -#define TK_AS 84 -#define TK_OUTPUTTYPE 85 -#define TK_AGGREGATE 86 -#define TK_BUFSIZE 87 -#define TK_PPS 88 -#define TK_TSERIES 89 -#define TK_DBS 90 -#define TK_STORAGE 91 -#define TK_QTIME 92 -#define TK_CONNS 93 -#define TK_STATE 94 -#define TK_COMMA 95 -#define TK_KEEP 96 -#define TK_CACHE 97 -#define TK_REPLICA 98 -#define TK_QUORUM 99 -#define TK_DAYS 100 -#define TK_MINROWS 101 -#define TK_MAXROWS 102 -#define TK_BLOCKS 103 -#define TK_CTIME 104 -#define TK_WAL 105 -#define TK_FSYNC 106 -#define TK_COMP 107 -#define TK_PRECISION 108 -#define TK_UPDATE 109 -#define TK_CACHELAST 110 -#define TK_STREAM 111 -#define TK_MODE 112 -#define TK_UNSIGNED 113 -#define TK_TAGS 114 -#define TK_USING 115 -#define TK_NULL 116 -#define TK_NOW 117 -#define TK_SELECT 118 -#define TK_UNION 119 -#define TK_ALL 120 -#define TK_DISTINCT 121 -#define TK_FROM 122 -#define TK_VARIABLE 123 -#define TK_INTERVAL 124 -#define TK_EVERY 125 -#define TK_SESSION 126 -#define TK_STATE_WINDOW 127 -#define TK_FILL 128 -#define TK_SLIDING 129 -#define TK_ORDER 130 -#define TK_BY 131 -#define TK_ASC 132 -#define TK_GROUP 133 -#define TK_HAVING 134 -#define TK_LIMIT 135 -#define TK_OFFSET 136 -#define TK_SLIMIT 137 -#define TK_SOFFSET 138 -#define TK_WHERE 139 -#define TK_RESET 140 -#define TK_QUERY 141 -#define TK_SYNCDB 142 -#define TK_ADD 143 -#define TK_COLUMN 144 -#define TK_MODIFY 145 -#define TK_TAG 146 -#define TK_CHANGE 147 -#define TK_SET 148 -#define TK_KILL 149 -#define TK_CONNECTION 150 -#define TK_COLON 151 -#define TK_ABORT 152 -#define TK_AFTER 153 -#define TK_ATTACH 154 -#define TK_BEFORE 155 -#define TK_BEGIN 156 -#define TK_CASCADE 157 -#define TK_CLUSTER 158 -#define TK_CONFLICT 159 -#define TK_COPY 160 -#define TK_DEFERRED 161 -#define TK_DELIMITERS 162 -#define TK_DETACH 163 -#define TK_EACH 164 -#define TK_END 165 -#define TK_EXPLAIN 166 -#define TK_FAIL 167 -#define TK_FOR 168 -#define TK_IGNORE 169 -#define TK_IMMEDIATE 170 -#define TK_INITIALLY 171 -#define TK_INSTEAD 172 -#define TK_KEY 173 -#define TK_OF 174 -#define TK_RAISE 175 -#define TK_REPLACE 176 -#define TK_RESTRICT 177 -#define TK_ROW 178 -#define TK_STATEMENT 179 -#define TK_TRIGGER 180 -#define TK_VIEW 181 -#define TK_SEMI 182 -#define TK_NONE 183 -#define TK_PREV 184 -#define TK_LINEAR 185 -#define TK_IMPORT 186 -#define TK_TBNAME 187 -#define TK_JOIN 188 -#define TK_INSERT 189 -#define TK_INTO 190 -#define TK_VALUES 191 +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_INTEGER 3 +#define TK_FLOAT 4 +#define TK_STRING 5 +#define TK_TIMESTAMP 6 +#define TK_OR 7 +#define TK_AND 8 +#define TK_NOT 9 +#define TK_EQ 10 +#define TK_NE 11 +#define TK_ISNULL 12 +#define TK_NOTNULL 13 +#define TK_IS 14 +#define TK_LIKE 15 +#define TK_MATCH 16 +#define TK_NMATCH 17 +#define TK_GLOB 18 +#define TK_BETWEEN 19 +#define TK_IN 20 +#define TK_GT 21 +#define TK_GE 22 +#define TK_LT 23 +#define TK_LE 24 +#define TK_BITAND 25 +#define TK_BITOR 26 +#define TK_LSHIFT 27 +#define TK_RSHIFT 28 +#define TK_PLUS 29 +#define TK_MINUS 30 +#define TK_DIVIDE 31 +#define TK_TIMES 32 +#define TK_STAR 33 +#define TK_SLASH 34 +#define TK_REM 35 +#define TK_CONCAT 36 +#define TK_UMINUS 37 +#define TK_UPLUS 38 +#define TK_BITNOT 39 +#define TK_SHOW 40 +#define TK_DATABASES 41 +#define TK_TOPICS 42 +#define TK_FUNCTIONS 43 +#define TK_MNODES 44 +#define TK_DNODES 45 +#define TK_ACCOUNTS 46 +#define TK_USERS 47 +#define TK_MODULES 48 +#define TK_QUERIES 49 +#define TK_CONNECTIONS 50 +#define TK_STREAMS 51 +#define TK_VARIABLES 52 +#define TK_SCORES 53 +#define TK_GRANTS 54 +#define TK_VNODES 55 +#define TK_DOT 56 +#define TK_CREATE 57 +#define TK_TABLE 58 +#define TK_STABLE 59 +#define TK_DATABASE 60 +#define TK_TABLES 61 +#define TK_STABLES 62 +#define TK_VGROUPS 63 +#define TK_DROP 64 +#define TK_TOPIC 65 +#define TK_FUNCTION 66 +#define TK_DNODE 67 +#define TK_USER 68 +#define TK_ACCOUNT 69 +#define TK_USE 70 +#define TK_DESCRIBE 71 +#define TK_DESC 72 +#define TK_ALTER 73 +#define TK_PASS 74 +#define TK_PRIVILEGE 75 +#define TK_LOCAL 76 +#define TK_COMPACT 77 +#define TK_LP 78 +#define TK_RP 79 +#define TK_IF 80 +#define TK_EXISTS 81 +#define TK_PORT 82 +#define TK_IPTOKEN 83 +#define TK_AS 84 +#define TK_OUTPUTTYPE 85 +#define TK_AGGREGATE 86 +#define TK_BUFSIZE 87 +#define TK_PPS 88 +#define TK_TSERIES 89 +#define TK_DBS 90 +#define TK_STORAGE 91 +#define TK_QTIME 92 +#define TK_CONNS 93 +#define TK_STATE 94 +#define TK_COMMA 95 +#define TK_KEEP 96 +#define TK_CACHE 97 +#define TK_REPLICA 98 +#define TK_QUORUM 99 +#define TK_DAYS 100 +#define TK_MINROWS 101 +#define TK_MAXROWS 102 +#define TK_BLOCKS 103 +#define TK_CTIME 104 +#define TK_WAL 105 +#define TK_FSYNC 106 +#define TK_COMP 107 +#define TK_PRECISION 108 +#define TK_UPDATE 109 +#define TK_CACHELAST 110 +#define TK_STREAM 111 +#define TK_MODE 112 +#define TK_UNSIGNED 113 +#define TK_TAGS 114 +#define TK_USING 115 +#define TK_NULL 116 +#define TK_NOW 117 +#define TK_SELECT 118 +#define TK_UNION 119 +#define TK_ALL 120 +#define TK_DISTINCT 121 +#define TK_FROM 122 +#define TK_VARIABLE 123 +#define TK_INTERVAL 124 +#define TK_EVERY 125 +#define TK_SESSION 126 +#define TK_STATE_WINDOW 127 +#define TK_FILL 128 +#define TK_SLIDING 129 +#define TK_ORDER 130 +#define TK_BY 131 +#define TK_ASC 132 +#define TK_GROUP 133 +#define TK_HAVING 134 +#define TK_LIMIT 135 +#define TK_OFFSET 136 +#define TK_SLIMIT 137 +#define TK_SOFFSET 138 +#define TK_WHERE 139 +#define TK_RESET 140 +#define TK_QUERY 141 +#define TK_SYNCDB 142 +#define TK_ADD 143 +#define TK_COLUMN 144 +#define TK_MODIFY 145 +#define TK_TAG 146 +#define TK_CHANGE 147 +#define TK_SET 148 +#define TK_KILL 149 +#define TK_CONNECTION 150 +#define TK_COLON 151 +#define TK_ABORT 152 +#define TK_AFTER 153 +#define TK_ATTACH 154 +#define TK_BEFORE 155 +#define TK_BEGIN 156 +#define TK_CASCADE 157 +#define TK_CLUSTER 158 +#define TK_CONFLICT 159 +#define TK_COPY 160 +#define TK_DEFERRED 161 +#define TK_DELIMITERS 162 +#define TK_DETACH 163 +#define TK_EACH 164 +#define TK_END 165 +#define TK_EXPLAIN 166 +#define TK_FAIL 167 +#define TK_FOR 168 +#define TK_IGNORE 169 +#define TK_IMMEDIATE 170 +#define TK_INITIALLY 171 +#define TK_INSTEAD 172 +#define TK_KEY 173 +#define TK_OF 174 +#define TK_RAISE 175 +#define TK_REPLACE 176 +#define TK_RESTRICT 177 +#define TK_ROW 178 +#define TK_STATEMENT 179 +#define TK_TRIGGER 180 +#define TK_VIEW 181 +#define TK_SEMI 182 +#define TK_NONE 183 +#define TK_PREV 184 +#define TK_LINEAR 185 +#define TK_IMPORT 186 +#define TK_TBNAME 187 +#define TK_JOIN 188 +#define TK_INSERT 189 +#define TK_INTO 190 +#define TK_VALUES 191 -#define NEW_TK_OR 1 -#define NEW_TK_AND 2 -#define NEW_TK_UNION 3 -#define NEW_TK_ALL 4 -#define NEW_TK_MINUS 5 -#define NEW_TK_EXCEPT 6 -#define NEW_TK_INTERSECT 7 -#define NEW_TK_NK_PLUS 8 -#define NEW_TK_NK_MINUS 9 -#define NEW_TK_NK_STAR 10 -#define NEW_TK_NK_SLASH 11 -#define NEW_TK_NK_REM 12 -#define NEW_TK_SHOW 13 -#define NEW_TK_DATABASES 14 -#define NEW_TK_NK_INTEGER 15 -#define NEW_TK_NK_FLOAT 16 -#define NEW_TK_NK_STRING 17 -#define NEW_TK_NK_BOOL 18 -#define NEW_TK_TIMESTAMP 19 -#define NEW_TK_NK_VARIABLE 20 -#define NEW_TK_NK_COMMA 21 -#define NEW_TK_NK_ID 22 -#define NEW_TK_NK_LP 23 -#define NEW_TK_NK_RP 24 -#define NEW_TK_NK_DOT 25 -#define NEW_TK_BETWEEN 26 -#define NEW_TK_NOT 27 -#define NEW_TK_IS 28 -#define NEW_TK_NULL 29 -#define NEW_TK_NK_LT 30 -#define NEW_TK_NK_GT 31 -#define NEW_TK_NK_LE 32 -#define NEW_TK_NK_GE 33 -#define NEW_TK_NK_NE 34 -#define NEW_TK_NK_EQ 35 -#define NEW_TK_LIKE 36 -#define NEW_TK_MATCH 37 -#define NEW_TK_NMATCH 38 -#define NEW_TK_IN 39 -#define NEW_TK_FROM 40 -#define NEW_TK_AS 41 -#define NEW_TK_JOIN 42 -#define NEW_TK_ON 43 -#define NEW_TK_INNER 44 -#define NEW_TK_SELECT 45 -#define NEW_TK_DISTINCT 46 -#define NEW_TK_WHERE 47 -#define NEW_TK_PARTITION 48 -#define NEW_TK_BY 49 -#define NEW_TK_SESSION 50 -#define NEW_TK_STATE_WINDOW 51 -#define NEW_TK_INTERVAL 52 -#define NEW_TK_SLIDING 53 -#define NEW_TK_FILL 54 -#define NEW_TK_VALUE 55 -#define NEW_TK_NONE 56 -#define NEW_TK_PREV 57 -#define NEW_TK_LINEAR 58 -#define NEW_TK_NEXT 59 -#define NEW_TK_GROUP 60 -#define NEW_TK_HAVING 61 -#define NEW_TK_ORDER 62 -#define NEW_TK_SLIMIT 63 -#define NEW_TK_SOFFSET 64 -#define NEW_TK_LIMIT 65 -#define NEW_TK_OFFSET 66 -#define NEW_TK_ASC 67 -#define NEW_TK_DESC 68 -#define NEW_TK_NULLS 69 -#define NEW_TK_FIRST 70 -#define NEW_TK_LAST 71 +#define NEW_TK_OR 1 +#define NEW_TK_AND 2 +#define NEW_TK_UNION 3 +#define NEW_TK_ALL 4 +#define NEW_TK_MINUS 5 +#define NEW_TK_EXCEPT 6 +#define NEW_TK_INTERSECT 7 +#define NEW_TK_NK_PLUS 8 +#define NEW_TK_NK_MINUS 9 +#define NEW_TK_NK_STAR 10 +#define NEW_TK_NK_SLASH 11 +#define NEW_TK_NK_REM 12 +#define NEW_TK_SHOW 13 +#define NEW_TK_DATABASES 14 +#define NEW_TK_NK_INTEGER 15 +#define NEW_TK_NK_FLOAT 16 +#define NEW_TK_NK_STRING 17 +#define NEW_TK_NK_BOOL 18 +#define NEW_TK_TIMESTAMP 19 +#define NEW_TK_NK_VARIABLE 20 +#define NEW_TK_NK_COMMA 21 +#define NEW_TK_NK_ID 22 +#define NEW_TK_NK_LP 23 +#define NEW_TK_NK_RP 24 +#define NEW_TK_NK_DOT 25 +#define NEW_TK_BETWEEN 26 +#define NEW_TK_NOT 27 +#define NEW_TK_IS 28 +#define NEW_TK_NULL 29 +#define NEW_TK_NK_LT 30 +#define NEW_TK_NK_GT 31 +#define NEW_TK_NK_LE 32 +#define NEW_TK_NK_GE 33 +#define NEW_TK_NK_NE 34 +#define NEW_TK_NK_EQ 35 +#define NEW_TK_LIKE 36 +#define NEW_TK_MATCH 37 +#define NEW_TK_NMATCH 38 +#define NEW_TK_IN 39 +#define NEW_TK_FROM 40 +#define NEW_TK_AS 41 +#define NEW_TK_JOIN 42 +#define NEW_TK_ON 43 +#define NEW_TK_INNER 44 +#define NEW_TK_SELECT 45 +#define NEW_TK_DISTINCT 46 +#define NEW_TK_WHERE 47 +#define NEW_TK_PARTITION 48 +#define NEW_TK_BY 49 +#define NEW_TK_SESSION 50 +#define NEW_TK_STATE_WINDOW 51 +#define NEW_TK_INTERVAL 52 +#define NEW_TK_SLIDING 53 +#define NEW_TK_FILL 54 +#define NEW_TK_VALUE 55 +#define NEW_TK_NONE 56 +#define NEW_TK_PREV 57 +#define NEW_TK_LINEAR 58 +#define NEW_TK_NEXT 59 +#define NEW_TK_GROUP 60 +#define NEW_TK_HAVING 61 +#define NEW_TK_ORDER 62 +#define NEW_TK_SLIMIT 63 +#define NEW_TK_SOFFSET 64 +#define NEW_TK_LIMIT 65 +#define NEW_TK_OFFSET 66 +#define NEW_TK_ASC 67 +#define NEW_TK_DESC 68 +#define NEW_TK_NULLS 69 +#define NEW_TK_FIRST 70 +#define NEW_TK_LAST 71 -#define TK_SPACE 300 -#define TK_COMMENT 301 -#define TK_ILLEGAL 302 -#define TK_HEX 303 // hex number 0x123 -#define TK_OCT 304 // oct number -#define TK_BIN 305 // bin format data 0b111 -#define TK_FILE 306 -#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query - -#define TK_NIL 65535 - -#endif +#define TK_SPACE 300 +#define TK_COMMENT 301 +#define TK_ILLEGAL 302 +#define TK_HEX 303 // hex number 0x123 +#define TK_OCT 304 // oct number +#define TK_BIN 305 // bin format data 0b111 +#define TK_FILE 306 +#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_NIL 65535 +#endif /*_TD_COMMON_TOKEN_H_*/ diff --git a/include/common/ttszip.h b/include/common/ttszip.h index 38699ae79197c1400a6a4c7995c01c7766f66d84..e9f430232054a4a13c8aec7eada9b452d4873ca8 100644 --- a/include/common/ttszip.h +++ b/include/common/ttszip.h @@ -13,19 +13,19 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTSZIP_H -#define TDENGINE_TTSZIP_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_COMMON_TTSZIP_H_ +#define _TD_COMMON_TTSZIP_H_ #include "os.h" #include "tdef.h" #include "tvariant.h" -#define MEM_BUF_SIZE (1 << 20) -#define TS_COMP_FILE_MAGIC 0x87F5EC4C +#ifdef __cplusplus +extern "C" { +#endif + +#define MEM_BUF_SIZE (1 << 20) +#define TS_COMP_FILE_MAGIC 0x87F5EC4C #define TS_COMP_FILE_GROUP_MAX 512 typedef struct STSList { @@ -61,10 +61,10 @@ typedef struct STSBlock { * and the offset of int32_t type is enough */ typedef struct STSGroupBlockInfo { - int32_t id; // group id - int32_t offset; // offset set value in file - int32_t numOfBlocks; // number of total blocks - int32_t compLen; // compressed size + int32_t id; // group id + int32_t offset; // offset set value in file + int32_t numOfBlocks; // number of total blocks + int32_t compLen; // compressed size } STSGroupBlockInfo; typedef struct STSGroupBlockInfoEx { @@ -73,9 +73,9 @@ typedef struct STSGroupBlockInfoEx { } STSGroupBlockInfoEx; typedef struct STSBuf { - FILE* f; - char path[PATH_MAX]; - uint32_t fileSize; + TdFilePtr pFile; + char path[PATH_MAX]; + uint32_t fileSize; // todo use array STSGroupBlockInfoEx* pData; @@ -112,9 +112,9 @@ STSBuf* tsBufClone(STSBuf* pTSBuf); STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id); -void tsBufFlush(STSBuf* pTSBuf); -void tsBufResetPos(STSBuf* pTSBuf); -bool tsBufNextPos(STSBuf* pTSBuf); +void tsBufFlush(STSBuf* pTSBuf); +void tsBufResetPos(STSBuf* pTSBuf); +bool tsBufNextPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag); @@ -144,4 +144,4 @@ bool tsBufIsValidElem(STSElem* pElem); } #endif -#endif // TDENGINE_TTSZIP_H +#endif /*_TD_COMMON_TTSZIP_H_*/ diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 39a5f0e2b8a42f96390b060929c9c4bcd10beeb6..5aa22bdcad699e7502d880786372d31a3545bd56 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -1,5 +1,20 @@ -#ifndef TDENGINE_TTYPE_H -#define TDENGINE_TTYPE_H +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_TTYPE_H_ +#define _TD_COMMON_TTYPE_H_ #ifdef __cplusplus extern "C" { @@ -15,11 +30,6 @@ typedef uint8_t TDRowValT; typedef uint16_t col_id_t; typedef int8_t col_type_t; -typedef struct tstr { - VarDataLenT len; - char data[]; -} tstr; - #pragma pack(push, 1) typedef struct { VarDataLenT len; @@ -32,19 +42,18 @@ typedef struct { } SNCharNullT; #pragma pack(pop) -#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) -#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) -#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) -#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) -#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) - -#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) -#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataCopy(dst, v) memcpy((dst), (void *)(v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char *)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT)(_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) +#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) +#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) // this data type is internally used only in 'in' query to hold the values -#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) -#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) +#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) +#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) #define GET_TYPED_DATA(_v, _finalType, _type, _data) \ do { \ @@ -62,7 +71,7 @@ typedef struct { case TSDB_DATA_TYPE_USMALLINT: \ (_v) = (_finalType)GET_UINT16_VAL(_data); \ break; \ - case TSDB_DATA_TYPE_TIMESTAMP:\ + case TSDB_DATA_TYPE_TIMESTAMP: \ case TSDB_DATA_TYPE_BIGINT: \ (_v) = (_finalType)(GET_INT64_VAL(_data)); \ break; \ @@ -91,7 +100,7 @@ typedef struct { do { \ switch (_type) { \ case TSDB_DATA_TYPE_BOOL: \ - *(bool *)(_v) = (bool)(_data); \ + *(bool *)(_v) = (bool)(_data); \ break; \ case TSDB_DATA_TYPE_TINYINT: \ *(int8_t *)(_v) = (int8_t)(_data); \ @@ -135,18 +144,19 @@ typedef struct { #define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) -#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) -#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) -#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) -#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) -#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) -#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) -#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) -#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) -#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) -#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) - -#define IS_CONVERT_AS_SIGNED(_t) (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) +#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) +#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) + +#define IS_CONVERT_AS_SIGNED(_t) \ + (IS_SIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) #define IS_CONVERT_AS_UNSIGNED(_t) (IS_UNSIGNED_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL)) static FORCE_INLINE bool isNull(const void *val, int32_t type) { @@ -167,17 +177,17 @@ static FORCE_INLINE bool isNull(const void *val, int32_t type) { case TSDB_DATA_TYPE_DOUBLE: return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; case TSDB_DATA_TYPE_NCHAR: - return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + return varDataLen(val) == sizeof(int32_t) && *(uint32_t *)varDataVal(val) == TSDB_DATA_NCHAR_NULL; case TSDB_DATA_TYPE_BINARY: - return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *)varDataVal(val) == TSDB_DATA_BINARY_NULL; case TSDB_DATA_TYPE_UTINYINT: - return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + return *(uint8_t *)val == TSDB_DATA_UTINYINT_NULL; case TSDB_DATA_TYPE_USMALLINT: - return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + return *(uint16_t *)val == TSDB_DATA_USMALLINT_NULL; case TSDB_DATA_TYPE_UINT: - return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + return *(uint32_t *)val == TSDB_DATA_UINT_NULL; case TSDB_DATA_TYPE_UBIGINT: - return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + return *(uint64_t *)val == TSDB_DATA_UBIGINT_NULL; default: return false; @@ -188,32 +198,31 @@ typedef struct tDataTypeDescriptor { int16_t type; int16_t nameLen; int32_t bytes; - char * name; + char *name; int64_t minValue; int64_t maxValue; - int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize); - int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize); - void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minindex, int16_t *maxindex, int16_t *numofnull); + int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, + int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[15]; bool isValidDataType(int32_t type); -void setVardataNull(void* val, int32_t type); -void setNull(void *val, int32_t type, int32_t bytes); -void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); +void setVardataNull(void *val, int32_t type); +void setNull(void *val, int32_t type, int32_t bytes); +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); const void *getNullValue(int32_t type); -void assignVal(char *val, const char *src, int32_t len, int32_t type); -void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); -void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); -void* getDataMin(int32_t type); -void* getDataMax(int32_t type); - +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf); +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); +void *getDataMin(int32_t type); +void *getDataMax(int32_t type); #define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) #define SET_BIGINT_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_BIGINT_NULL) @@ -222,4 +231,4 @@ void* getDataMax(int32_t type); } #endif -#endif // TDENGINE_TTYPE_H +#endif /*_TD_COMMON_TTYPE_H_*/ diff --git a/include/common/tvariant.h b/include/common/tvariant.h index e068c6eb6ca4cc56b3eae27d5ca213517ad86752..995015fe6301b950f2ae415aef791296e6621d28 100644 --- a/include/common/tvariant.h +++ b/include/common/tvariant.h @@ -13,34 +13,34 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TVARIANT_H -#define TDENGINE_TVARIANT_H +#ifndef _TD_COMMON_VARIANT_H_ +#define _TD_COMMON_VARIANT_H_ + +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#include "tarray.h" - // variant, each number/string/field_id has a corresponding struct during parsing sql typedef struct SVariant { - uint32_t nType; - int32_t nLen; // only used for string, for number, it is useless + uint32_t nType; + int32_t nLen; // only used for string, for number, it is useless union { int64_t i; uint64_t u; double d; char *pz; wchar_t *wpz; - SArray *arr; // only for 'in' query to hold value list, not value for a field + SArray *arr; // only for 'in' query to hold value list, not value for a field }; } SVariant; -int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* issigned); +int32_t toInteger(const char *z, int32_t n, int32_t base, int64_t *value, bool *issigned); bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, const char* z, int32_t n, int32_t type); +void taosVariantCreate(SVariant *pVar, const char *z, int32_t n, int32_t type); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); @@ -48,7 +48,7 @@ void taosVariantDestroy(SVariant *pV); void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); -int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2); +int32_t taosVariantCompare(const SVariant *p1, const SVariant *p2); int32_t taosVariantToString(SVariant *pVar, char *dst); @@ -64,4 +64,4 @@ int32_t taosVariantTypeSetType(SVariant *pVariant, char type); } #endif -#endif // TDENGINE_TVARIANT_H +#endif /*_TD_COMMON_VARIANT_H_*/ diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index e64d358de442be20ebf07d85b0ec4597c07ef14b..068e9a69170bfd1659b2961bef220b765d74c35e 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -25,26 +25,12 @@ extern "C" { /* ------------------------ TYPES EXPOSED ---------------- */ typedef struct SDnode SDnode; -/* ------------------------ Environment ------------------ */ -typedef struct { - int32_t sver; - int32_t numOfCores; - int16_t numOfCommitThreads; - int8_t enableTelem; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; -} SDnodeEnvCfg; - /** * @brief Initialize the environment * - * @param pOption Option of the environment * @return int32_t 0 for success and -1 for failure */ -int32_t dndInit(const SDnodeEnvCfg *pCfg); +int32_t dndInit(); /** * @brief clear the environment @@ -54,17 +40,15 @@ void dndCleanup(); /* ------------------------ SDnode ----------------------- */ typedef struct { - int32_t numOfSupportVnodes; - int32_t statusInterval; - float numOfThreadsPerCore; - float ratioOfQueryCores; - int32_t maxShellConns; - int32_t shellActivityTimer; - uint16_t serverPort; - char dataDir[TSDB_FILENAME_LEN]; - char localEp[TSDB_EP_LEN]; - char localFqdn[TSDB_FQDN_LEN]; - char firstEp[TSDB_EP_LEN]; + int32_t numOfSupportVnodes; + uint16_t serverPort; + char dataDir[TSDB_FILENAME_LEN]; + char localEp[TSDB_EP_LEN]; + char localFqdn[TSDB_FQDN_LEN]; + char firstEp[TSDB_EP_LEN]; + char secondEp[TSDB_EP_LEN]; + SDiskCfg *pDisks; + int32_t numOfDisks; } SDnodeObjCfg; /** diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index e1306800d3e3c9491bad297130d23ba83932ec6e..2680bb83edfca41187150f3f04daf4fa7c369017 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -44,25 +44,12 @@ typedef struct SMnodeLoad { int64_t compStorage; } SMnodeLoad; -typedef struct SMnodeCfg { - int32_t sver; - int8_t enableTelem; - int32_t statusInterval; - int32_t shellActivityTimer; - char *timezone; - char *locale; - char *charset; - char *buildinfo; - char *gitinfo; -} SMnodeCfg; - typedef struct { int32_t dnodeId; int64_t clusterId; int8_t replica; int8_t selfIndex; SReplica replicas[TSDB_MAX_REPLICA]; - SMnodeCfg cfg; SDnode *pDnode; PutReqToMWriteQFp putReqToMWriteQFp; PutReqToMReadQFp putReqToMReadQFp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b6054c02b4ae54927c5f0e7ff1d41adc33243e6b..ebe7c92d31b26779f4f478c332acafc862ba71a7 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -24,7 +24,7 @@ extern "C" { #include "taosdef.h" #include "query.h" #include "tname.h" -#include "common.h" +#include "tcommon.h" #include "tarray.h" #include "thash.h" #include "tmsg.h" diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d9e1957e5ddbb21d8e49786ea11901c06f3087ee..a4c0be27abeae8566aa5efabfb20972107e70b2c 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" typedef void* qTaskInfo_t; typedef void* DataSinkHandle; @@ -31,6 +31,7 @@ typedef struct SReadHandle { void* reader; void* meta; } SReadHandle; + /** * Create the exec task for streaming mode * @param pMsg @@ -40,12 +41,22 @@ typedef struct SReadHandle { qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); /** - * + * Set the input data block for the stream scan. * @param tinfo * @param input * @return */ -int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); + +/** + * Update the table id list, add or remove. + * + * @param tinfo + * @param id + * @param isAdd + * @return + */ +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json diff --git a/include/libs/function/function.h b/include/libs/function/function.h index e970a0d693db322a4cfa8a38e5aa28f778ab4197..62063ef99a67b961b852a3870600d4d4b850ed7b 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "tvariant.h" #include "tbuffer.h" @@ -138,8 +138,10 @@ extern SFunctionFpSet fpSet[1]; // sql function runtime context typedef struct SqlFunctionCtx { + int32_t startRow; int32_t size; // number of rows - void * pInput; // input data buffer + SColumnInfoData* pInput; + uint32_t order; // asc|desc int16_t inputType; int16_t inputBytes; @@ -227,11 +229,16 @@ typedef struct SAggFunctionInfo { } SAggFunctionInfo; typedef struct SScalarParam { - void* data; - bool colData; - int32_t num; - int32_t type; - int32_t bytes; + void *data; + union { + SColumnInfoData *columnData; + void *data; + } orig; + char *bitmap; + bool dataInBlock; + int32_t num; + int32_t type; + int32_t bytes; } SScalarParam; typedef struct SScalarFunctionInfo { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index f09ffa8926da51396e1f32161e0b17c7bf8aaed9..96815ac29fe583b430e22afe094f3fa6ec25aeaf 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -62,9 +62,8 @@ typedef enum ENodeType { QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, QUERY_NODE_RAW_EXPR, // Only be used in parser module. - QUERY_NODE_COLUMN_REF, QUERY_NODE_TARGET, - QUERY_NODE_TUPLE_DESC, + QUERY_NODE_DATABLOCK_DESC, QUERY_NODE_SLOT_DESC, // Statement nodes are used in parser and planner module. @@ -81,7 +80,9 @@ typedef enum ENodeType { // physical plan node QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_PROJECT + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + QUERY_NODE_PHYSICAL_PLAN_JOIN, + QUERY_NODE_PHYSICAL_PLAN_AGG } ENodeType; /** diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 40684f2b2622afff6d8cbf781222e15c92d84d65..608146e3df99942bafdab181c9cc484a781e0f26 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -69,21 +69,20 @@ typedef struct SSlotDescNode { ENodeType type; int16_t slotId; SDataType dataType; - int16_t srcTupleId; - int16_t srcSlotId; bool reserve; bool output; + bool tag; } SSlotDescNode; -typedef struct STupleDescNode { +typedef struct SDataBlockDescNode { ENodeType type; - int16_t tupleId; + int16_t dataBlockId; SNodeList* pSlots; -} STupleDescNode; +} SDataBlockDescNode; typedef struct SPhysiNode { ENodeType type; - STupleDescNode outputTuple; + SDataBlockDescNode outputDataBlockDesc; SNode* pConditions; SNodeList* pChildren; struct SPhysiNode* pParent; @@ -106,6 +105,7 @@ typedef struct STableScanPhysiNode { SScanPhysiNode scan; uint8_t scanFlag; // denotes reversed scan of data or not STimeWindow scanRange; + SNode* pScanConditions; } STableScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode; @@ -115,6 +115,20 @@ typedef struct SProjectPhysiNode { SNodeList* pProjections; } SProjectPhysiNode; +typedef struct SJoinPhysiNode { + SPhysiNode node; + EJoinType joinType; + SNode* pOnConditions; // in or out tuple ? + SNodeList* pTargets; +} SJoinPhysiNode; + +typedef struct SAggPhysiNode { + SPhysiNode node; + SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function + SNodeList* pGroupKeys; // SColumnRefNode list + SNodeList* pAggFuncs; +} SAggPhysiNode; + #ifdef __cplusplus } #endif diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 7bc8617db18626b140880abeca291b36e44c24e9..42f9310ef1d4221aba543651fdd9bcf8f278b18f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -58,19 +58,13 @@ typedef struct SColumnNode { char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; SNode* pProjectRef; -} SColumnNode; - -typedef struct SColumnRefNode { - ENodeType type; - SDataType dataType; - int16_t tupleId; + int16_t dataBlockId; int16_t slotId; - int16_t columnId; -} SColumnRefNode; +} SColumnNode; typedef struct STargetNode { ENodeType type; - int16_t tupleId; + int16_t dataBlockId; int16_t slotId; SNode* pExpr; } STargetNode; diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 6cd5feca2400465e0543c7873ed00388d06abcb8..0d209a7f4d579b834c50cb14808c20bd63576827 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "catalog.h" -#include "common.h" +#include "tcommon.h" #include "function.h" #include "tmsgtype.h" #include "tname.h" diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 9030ffc9467af30f5dd058fb298bac619779c828..af8a7dccd45c2e4bc7370b6b077d4653943b4371 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -47,5 +47,6 @@ OP_ENUM_MACRO(AllTimeWindow) OP_ENUM_MACRO(AllMultiTableTimeInterval) OP_ENUM_MACRO(Order) OP_ENUM_MACRO(Exchange) +OP_ENUM_MACRO(SortedMerge) //OP_ENUM_MACRO(TableScan) diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index fd57016ec3491f354fe05a063b9d054a854c7993..fedb487931a6e721a49ba0194716f457aa1b433e 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -37,8 +37,8 @@ typedef struct SFilterColumnParam{ extern int32_t filterInitFromNode(SNode *pNode, SFilterInfo **pinfo, uint32_t options); extern bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols); -extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp); -extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp); +extern int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param); +extern int32_t filterSetDataFromColId(SFilterInfo *info, void *param); extern int32_t filterGetTimeRange(SFilterInfo *info, STimeWindow *win); extern int32_t filterConverNcharColumns(SFilterInfo* pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo* pFilterInfo); diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index 1902c11cc68097b63141d640ce7a8bdf4656d713..e55c0b9e7667e934236c771d8980ace53aa57ca9 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -25,9 +25,17 @@ extern "C" { typedef struct SFilterInfo SFilterInfo; - +/* +pNode will be freed in API; +*pRes need to freed in caller +*/ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes); -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst); + +/* +pDst need to freed in caller +*/ +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst); + int32_t scalarGetOperatorParamNum(EOperatorType type); int32_t scalarGenerateSetFromList(void **data, void *pNode, uint32_t type); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 0ec741ec3e2ab30f7079ae6c7b266b65406c714b..a619e66622dfc61b4d8ba504d53e60313b0ef806 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -21,7 +21,9 @@ extern "C" { #endif #include +#include #include "taosdef.h" +#include "trpc.h" typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; @@ -34,23 +36,23 @@ typedef enum { TAOS_SYNC_STATE_LEADER = 2, } ESyncState; -typedef struct { +typedef struct SSyncBuffer { void* data; size_t len; } SSyncBuffer; -typedef struct { - SyncNodeId nodeId; - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN +typedef struct SNodeInfo { + uint16_t nodePort; // node sync Port + char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN } SNodeInfo; -typedef struct { +typedef struct SSyncCfg { int32_t replicaNum; + int32_t myIndex; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; } SSyncCfg; -typedef struct { +typedef struct SNodesRole { int32_t replicaNum; SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; ESyncState role[TSDB_MAX_REPLICA]; @@ -128,12 +130,12 @@ typedef struct SStateMgr { } SStateMgr; -typedef struct { - SyncGroupId vgId; - SSyncCfg syncCfg; - SSyncLogStore logStore; - SStateMgr stateManager; - SSyncFSM syncFsm; +typedef struct SSyncInfo { + SyncGroupId vgId; + SSyncCfg syncCfg; + char path[TSDB_FILENAME_LEN]; + SSyncFSM* pFsm; + int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); } SSyncInfo; diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 289cfd9b04bfaf34fc4e8bb1ad5e07cc00ba4959..b6d10c81bffb230c5e8408d4773cd8ceccf25baa 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -16,7 +16,7 @@ #ifndef _TD_TFS_H_ #define _TD_TFS_H_ -#include "tcfg.h" +#include "tdef.h" #ifdef __cplusplus extern "C" { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 538aeb1a0e7927469cf90721dbbc16140c181e93..5e3860822ec161ef52eece2931b7c71dee4fb40d 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -78,12 +78,15 @@ typedef struct SRpcInit { // call back to retrieve the client auth info, for server app only int (*afp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); + // call back to keep conn or not + bool (*pfp)(void *parent, tmsg_t msgType); + void *parent; } SRpcInit; int32_t rpcInit(); void rpcCleanup(); -void * rpcOpen(const SRpcInit *pRpc); +void *rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 45f1d88c30a6421e752368a36caed1bd42427b13..f90dbb97fe1b2860cd103e878ba25787be7102ab 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -120,8 +120,8 @@ typedef struct SWal { int32_t fsyncSeq; // meta SWalVer vers; - int64_t writeLogTfd; - int64_t writeIdxTfd; + TdFilePtr pWriteLogTFile; + TdFilePtr pWriteIdxTFile; int32_t writeCur; SArray *fileInfoSet; // status @@ -138,8 +138,8 @@ typedef struct SWal { typedef struct SWalReadHandle { SWal *pWal; - int64_t readLogTfd; - int64_t readIdxTfd; + TdFilePtr pReadLogTFile; + TdFilePtr pReadIdxTFile; int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; diff --git a/include/os/os.h b/include/os/os.h index 023d2b04705074abcec4c1dc927211bf63512112..f020af5a657dcb3bcaf997ba0a68e6e57ca31b23 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_H -#define TDENGINE_OS_H +#ifndef _TD_OS_H_ +#define _TD_OS_H_ #ifdef __cplusplus extern "C" { @@ -59,6 +59,7 @@ extern "C" { #include "osEndian.h" #include "osEnv.h" #include "osFile.h" +#include "osLocale.h" #include "osLz4.h" #include "osMath.h" #include "osMemory.h" @@ -73,6 +74,7 @@ extern "C" { #include "osThread.h" #include "osTime.h" #include "osTimer.h" +#include "osTimezone.h" void osInit(); @@ -80,4 +82,4 @@ void osInit(); } #endif -#endif +#endif /*_TD_OS_H_*/ diff --git a/include/os/osDef.h b/include/os/osDef.h index 07d360a7c00a1c062e87d9512b91d86da203ce56..339bf13343a9b98d7fa9b3814bdb2a361b0b912c 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,6 +191,10 @@ extern "C" { #define TD_DIRSEP "/" #endif +#define TD_LOCALE_LEN 64 +#define TD_CHARSET_LEN 64 +#define TD_TIMEZONE_LEN 96 + #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 35c294af39bf49967ab6fa1f15bc3bf35a523911..e328c420638eb1d146abe97f8abf6bacb6286377 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -23,8 +23,8 @@ extern "C" { void taosRemoveDir(const char *dirname); int32_t taosDirExist(char *dirname); int32_t taosMkDir(const char *dirname); -void taosRemoveOldFiles(char *dirname, int32_t keepDays); -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen); +void taosRemoveOldFiles(const char *dirname, int32_t keepDays); +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, int32_t maxlen); #ifdef __cplusplus diff --git a/include/os/osEnv.h b/include/os/osEnv.h index a7fd86776c06ea6f399f6094ac52a78c0345a7e5..4ac073f6c22d485e87b7b41daebb402d49d865b9 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -16,15 +16,37 @@ #ifndef _TD_OS_ENV_H_ #define _TD_OS_ENV_H_ +#include "osSysinfo.h" + #ifdef __cplusplus extern "C" { #endif -extern char tsOsName[]; +extern char tsOsName[]; +extern char tsTimezone[]; +extern char tsCharset[]; +extern char tsLocale[]; +extern int8_t tsDaylight; +extern bool tsEnableCoreFile; +extern int64_t tsPageSize; +extern int64_t tsOpenMax; +extern int64_t tsStreamMax; +extern int32_t tsNumOfCores; +extern int32_t tsTotalMemoryMB; + +extern char configDir[]; extern char tsDataDir[]; extern char tsLogDir[]; -extern char tsScriptDir[]; -extern char configDir[]; +extern char tsTempDir[]; + +extern SDiskSpace tsDataSpace; +extern SDiskSpace tsLogSpace; +extern SDiskSpace tsTempSpace; + +void osInit(); +void osUpdate(); +bool osLogSpaceAvailable(); +void osSetTimezone(const char *timezone); #ifdef __cplusplus } diff --git a/include/os/osFile.h b/include/os/osFile.h index ac399fa3be73bbf9ed8564c17b31f59fd81d1d90..cbd5d693c5aea2d8faa000b006a6a08575932391 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -22,49 +22,64 @@ extern "C" { #include "osSocket.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -typedef int32_t FileFd; -#else -typedef int32_t FileFd; +#ifndef ALLOW_FORBID_FUNC + #define open OPEN_FUNC_TAOS_FORBID + #define fopen FOPEN_FUNC_TAOS_FORBID + // #define close CLOSE_FUNC_TAOS_FORBID + // #define fclose FCLOSE_FUNC_TAOS_FORBID #endif -#define FD_INITIALIZER ((int32_t)-1) - #ifndef PATH_MAX #define PATH_MAX 256 #endif -int32_t taosLockFile(FileFd fd); -int32_t taosUnLockFile(FileFd fd); - -int32_t taosUmaskFile(FileFd fd); - +typedef struct TdFile *TdFilePtr; + +#define TD_FILE_CTEATE 0x0001 +#define TD_FILE_WRITE 0x0002 +#define TD_FILE_READ 0x0004 +#define TD_FILE_TRUNC 0x0008 +#define TD_FILE_APPEND 0x0010 +#define TD_FILE_TEXT 0x0020 +#define TD_FILE_AUTO_DEL 0x0040 +#define TD_FILE_EXCL 0x0080 +#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosGetLineFile, taosEOFFile + +int32_t taosLockFile(TdFilePtr pFile); +int32_t taosUnLockFile(TdFilePtr pFile); + +int32_t taosUmaskFile(int32_t maskVal); + int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); -int32_t taosFStatFile(FileFd fd, int64_t *size, int32_t *mtime); - -FileFd taosOpenFileWrite(const char *path); -FileFd taosOpenFileCreateWrite(const char *path); -FileFd taosOpenFileCreateWriteTrunc(const char *path); -FileFd taosOpenFileCreateWriteAppend(const char *path); -FileFd taosOpenFileRead(const char *path); -FileFd taosOpenFileReadWrite(const char *path); - -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence); -int32_t taosFtruncateFile(FileFd fd, int64_t length); -int32_t taosFsyncFile(FileFd fd); - -int64_t taosReadFile(FileFd fd, void *buf, int64_t count); -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t count); - -void taosCloseFile(FileFd fd); - +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); + +TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions); + +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence); +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length); +int32_t taosFsyncFile(TdFilePtr pFile); + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count); +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset); +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...); +int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict__ ptrBuf); +int32_t taosEOFFile(TdFilePtr pFile); + +int64_t taosCloseFile(TdFilePtr *ppFile); + int32_t taosRenameFile(const char *oldName, const char *newName); int64_t taosCopyFile(const char *from, const char *to); - + void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath); + +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size); +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); + +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); +bool taosValidFile(TdFilePtr pFile); -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size); -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size); +int taosGetErrorFile(TdFilePtr pFile); #ifdef __cplusplus } diff --git a/include/os/osLocale.h b/include/os/osLocale.h new file mode 100644 index 0000000000000000000000000000000000000000..6e313eb8cde14e67f56a9e8334239ca714b88afd --- /dev/null +++ b/include/os/osLocale.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_LOCALE_H_ +#define _TD_OS_LOCALE_H_ + +#include "os.h" +#include "osString.h" + +#ifdef __cplusplus +extern "C" { +#endif + +char *taosCharsetReplace(char *charsetstr); +void taosGetSystemLocale(char *outLocale, char *outCharset); +void taosSetSystemLocale(const char *inLocale, const char *inCharSet); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_LOCALE_H_*/ diff --git a/include/os/osMemory.h b/include/os/osMemory.h index 5f1d5a9a8a6a7028295cac28d6711f6171be7307..6100419035ff9901bf6a6d5c3871496d0edcd533 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -20,12 +20,12 @@ extern "C" { #endif -#define tfree(x) \ - do { \ - if (x) { \ +#define tfree(x) \ + do { \ + if (x) { \ free((void *)(x)); \ (x) = 0; \ - } \ + } \ } while (0) #ifdef __cplusplus diff --git a/include/os/osSemaphore.h b/include/os/osSemaphore.h index 78112fc7a0f58d0659cdbd0043c47e814ca69ca0..7fb9a2202d38760e5f1c760fe9d7f8eaa4096f46 100644 --- a/include/os/osSemaphore.h +++ b/include/os/osSemaphore.h @@ -16,12 +16,12 @@ #ifndef _TD_OS_SEMPHONE_H_ #define _TD_OS_SEMPHONE_H_ +#include + #ifdef __cplusplus extern "C" { #endif -#include - #if defined (_TD_DARWIN_64) typedef struct tsem_s *tsem_t; int tsem_init(tsem_t *sem, int pshared, unsigned int value); diff --git a/include/os/osSocket.h b/include/os/osSocket.h index d76b286d651b0124fd5db4fd4018f566f498a0d0..395874a88c5bded90fad93e781ba187e7ec911b1 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -16,10 +16,6 @@ #ifndef _TD_OS_SOCKET_H_ #define _TD_OS_SOCKET_H_ -#ifdef __cplusplus -extern "C" { -#endif - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include "winsock2.h" #include @@ -30,6 +26,10 @@ extern "C" { #include #endif +#ifdef __cplusplus +extern "C" { +#endif + #define TAOS_EPOLL_WAIT_TIME 500 typedef int32_t SOCKET; typedef SOCKET EpollFd; @@ -55,7 +55,7 @@ void taosSetMaskSIGPIPE(); int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen); -uint32_t taosInetAddr(char *ipAddr); +uint32_t taosInetAddr(const char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) diff --git a/include/os/osString.h b/include/os/osString.h index 582411d4440e7b1539c4b926413c7ed9751dfe62..88160dd69e419450d1ec481a83a28065df99186d 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -45,7 +45,6 @@ int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, int32_t *len); int32_t tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int32_t bytes, int8_t ncharSize); bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); #ifdef __cplusplus } diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 9dcb075489ac3fed02c3bdb77610a420800c6859..a0771dc7340562787a0e8bd69fbb880998fb8dcc 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -16,30 +16,23 @@ #ifndef _TD_OS_SYSINFO_H_ #define _TD_OS_SYSINFO_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -#define TSDB_LOCALE_LEN 64 -#define TSDB_TIMEZONE_LEN 96 - -extern int64_t tsPageSize; -extern int64_t tsOpenMax; -extern int64_t tsStreamMax; -extern int32_t tsNumOfCores; -extern int32_t tsTotalMemoryMB; -extern char tsTimezone[]; -extern char tsLocale[]; -extern char tsCharset[]; // default encode string - typedef struct { int64_t total; int64_t used; int64_t avail; } SDiskSize; +typedef struct { + int64_t reserved; + SDiskSize size; +} SDiskSpace; + int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize); int32_t taosGetCpuCores(); void taosGetSystemInfo(); @@ -51,7 +44,6 @@ void taosGetDisk(); bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage); bool taosGetProcMemory(float *memoryUsedMB); bool taosGetSysMemory(float *memoryUsedMB); -void taosPrintOsInfo(); int taosSystem(const char *cmd); void taosKillSystem(); int32_t taosGetSystemUUID(char *uid, int32_t uidlen); diff --git a/include/os/osThread.h b/include/os/osThread.h index 79834dc9a59f00c0aefa452c2427ea45ac35c478..cccc13755d3a33f0ecaa1173df391228d3c90da9 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -16,12 +16,12 @@ #ifndef _TD_OS_THREAD_H_ #define _TD_OS_THREAD_H_ +#include + #ifdef __cplusplus extern "C" { #endif -#include - #ifdef __cplusplus } #endif diff --git a/include/common/tlocale.h b/include/os/osTimezone.h similarity index 75% rename from include/common/tlocale.h rename to include/os/osTimezone.h index d809c75d67983fb6ac844a656a361876ea7d3504..ff015ef0b1221eeba361818f5e010bd7f43fb0fd 100644 --- a/include/common/tlocale.h +++ b/include/os/osTimezone.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_LOCALE_H_ -#define _TD_COMMON_LOCALE_H_ +#ifndef _TD_OS_TIMEZONE_H_ +#define _TD_OS_TIMEZONE_H_ #ifdef __cplusplus extern "C" { #endif -void tsSetLocale(); +void taosGetSystemTimezone(char *outTimezone); +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight); #ifdef __cplusplus } -#endif /*_TD_COMMON_LOCALE_H_*/ - #endif + +#endif /*_TD_OS_TIMEZONE_H_*/ diff --git a/include/util/exception.h b/include/util/exception.h deleted file mode 100644 index 8012e3431591099a645373092fb34ac0fb66ea77..0000000000000000000000000000000000000000 --- a/include/util/exception.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2020 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_EXCEPTION_H -#define _TD_UTIL_EXCEPTION_H - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * cleanup actions - */ -typedef struct SCleanupAction { - bool failOnly; - uint8_t wrapper; - uint16_t reserved; - void* func; - union { - void* Ptr; - bool Bool; - char Char; - int8_t Int8; - uint8_t Uint8; - int16_t Int16; - uint16_t Uint16; - int Int; - unsigned int Uint; - int32_t Int32; - uint32_t Uint32; - int64_t Int64; - uint64_t Uint64; - float Float; - double Double; - } arg1, arg2; -} SCleanupAction; - - -/* - * exception hander registration - */ -typedef struct SExceptionNode { - struct SExceptionNode* prev; - jmp_buf jb; - int32_t code; - int32_t maxCleanupAction; - int32_t numCleanupAction; - SCleanupAction* cleanupActions; -} SExceptionNode; - -//////////////////////////////////////////////////////////////////////////////// -// functions & macros for auto-cleanup - -void cleanupPush_void_ptr_ptr ( bool failOnly, void* func, void* arg1, void* arg2 ); -void cleanupPush_void_ptr_bool ( bool failOnly, void* func, void* arg1, bool arg2 ); -void cleanupPush_void_ptr ( bool failOnly, void* func, void* arg ); -void cleanupPush_int_int ( bool failOnly, void* func, int arg ); -void cleanupPush_void ( bool failOnly, void* func ); -void cleanupPush_int_ptr ( bool failOnly, void* func, void* arg ); - -int32_t cleanupGetActionCount(); -void cleanupExecuteTo( int32_t anchor, bool failed ); -void cleanupExecute( SExceptionNode* node, bool failed ); -bool cleanupExceedLimit(); - -#define CLEANUP_PUSH_VOID_PTR_PTR( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_ptr( (failOnly), (void*)(func), (void*)(arg1), (void*)(arg2) ) -#define CLEANUP_PUSH_VOID_PTR_BOOL( failOnly, func, arg1, arg2 ) cleanupPush_void_ptr_bool( (failOnly), (void*)(func), (void*)(arg1), (bool)(arg2) ) -#define CLEANUP_PUSH_VOID_PTR( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (void*)(arg) ) -#define CLEANUP_PUSH_INT_INT( failOnly, func, arg ) cleanupPush_void_ptr( (failOnly), (void*)(func), (int)(arg) ) -#define CLEANUP_PUSH_VOID( failOnly, func ) cleanupPush_void( (failOnly), (void*)(func) ) -#define CLEANUP_PUSH_INT_PTR( failOnly, func, arg ) cleanupPush_int_ptr( (failOnly), (void*)(func), (void*)(arg) ) -#define CLEANUP_PUSH_FREE( failOnly, arg ) cleanupPush_void_ptr( (failOnly), free, (void*)(arg) ) -#define CLEANUP_PUSH_CLOSE( failOnly, arg ) cleanupPush_int_int( (failOnly), close, (int)(arg) ) -#define CLEANUP_PUSH_FCLOSE( failOnly, arg ) cleanupPush_int_ptr( (failOnly), fclose, (void*)(arg) ) - -#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() -#define CLEANUP_EXECUTE_TO( anchor, failed ) cleanupExecuteTo( (anchor), (failed) ) -#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit() - -//////////////////////////////////////////////////////////////////////////////// -// functions & macros for exception handling - -void exceptionPushNode( SExceptionNode* node ); -int32_t exceptionPopNode(); -void exceptionThrow( int32_t code ); - -#define TRY(maxCleanupActions) do { \ - SExceptionNode exceptionNode = { 0 }; \ - SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \ - exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \ - exceptionNode.cleanupActions = cleanupActions; \ - exceptionPushNode( &exceptionNode ); \ - int caughtException = setjmp( exceptionNode.jb ); \ - if( caughtException == 0 ) - -#define CATCH( code ) int32_t code = exceptionPopNode(); \ - if( caughtException == 1 ) - -#define FINALLY( code ) int32_t code = exceptionPopNode(); - -#define END_TRY } while( 0 ); - -#define THROW( x ) exceptionThrow( (x) ) -#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1)) -#define CLEANUP_EXECUTE() cleanupExecute( &exceptionNode, CAUGHT_EXCEPTION() ) - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_EXCEPTION_H*/ diff --git a/include/util/talgo.h b/include/util/talgo.h index 575f98b486f90b2eb4a830d0bb39868fcdc0c137..0fd44a6e91d664580a23188061cf14b9637919ab 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -13,16 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_TALGO_H -#define _TD_UTIL_TALGO_H +#ifndef _TD_UTIL_TALGO_H_ +#define _TD_UTIL_TALGO_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif #ifndef __COMPAR_FN_T -# define __COMPAR_FN_T -typedef int (*__compar_fn_t) (const void *, const void *); +#define __COMPAR_FN_T +typedef int32_t (*__compar_fn_t)(const void *, const void *); #endif #define TD_EQ 0x1 @@ -45,7 +47,7 @@ typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param); * @param param * @param comparFn */ -void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn); +void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn); /** * binary search, with range support @@ -58,7 +60,7 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex * @param flags * @return */ -void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags); +void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t fn, int32_t flags); /** * adjust heap @@ -74,7 +76,8 @@ void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, * @param maxroot: if heap is max root heap * @return */ -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -89,10 +92,11 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @param maxroot: if heap is max root heap * @return */ -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); - +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + const void *parswap, __ext_swap_fn_t swap, bool maxroot); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_TALGO_H*/ + +#endif /*_TD_UTIL_TALGO_H_*/ diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b947142958e555ffe523de2db4e6be9fd030907c..5fe69284a235b2ff19b5a0c604da80db68152b96 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TAOS_ERROR_H_ -#define _TD_COMMON_TAOS_ERROR_H_ +#ifndef _TD_UTIL_TAOS_ERROR_H_ +#define _TD_UTIL_TAOS_ERROR_H_ #ifdef __cplusplus extern "C" { @@ -38,28 +38,28 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FAILED -1 // unknown or needn't tell detail error // rpc -#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) //"Action in progress") -#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) //"Authentication required") -#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) //"Authentication failure") -#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) //"Redirect") -#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) //"System not ready") // peer is not ready to process data -#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) //"Message already processed") -#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) //"Last session not finished") -#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) //"Mismatched meter id") -#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) //"Processing of request timed out") -#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) //"Number of sessions reached limit") // too many sessions -#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) //"Unable to establish connection") -#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) //"Unexpected generic error in RPC") -#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) //"Unexpected response") -#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) //"Invalid value") -#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) //"Invalid transaction id") -#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) //"Invalid session id") -#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) //"Invalid message type") -#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) //"Invalid response type") -#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) //"Client and server's time is not synchronized") -#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) //"Database not ready") -#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) //"Unable to resolve FQDN") -#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //"Invalid app version") +#define TSDB_CODE_RPC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0001) +#define TSDB_CODE_RPC_AUTH_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x0002) +#define TSDB_CODE_RPC_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0003) +#define TSDB_CODE_RPC_REDIRECT TAOS_DEF_ERROR_CODE(0, 0x0004) +#define TSDB_CODE_RPC_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0005) +#define TSDB_CODE_RPC_ALREADY_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0006) +#define TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x0007) +#define TSDB_CODE_RPC_MISMATCHED_LINK_ID TAOS_DEF_ERROR_CODE(0, 0x0008) +#define TSDB_CODE_RPC_TOO_SLOW TAOS_DEF_ERROR_CODE(0, 0x0009) +#define TSDB_CODE_RPC_MAX_SESSIONS TAOS_DEF_ERROR_CODE(0, 0x000A) +#define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x000B) +#define TSDB_CODE_RPC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x000C) +#define TSDB_CODE_RPC_UNEXPECTED_RESPONSE TAOS_DEF_ERROR_CODE(0, 0x000D) +#define TSDB_CODE_RPC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x000E) +#define TSDB_CODE_RPC_INVALID_TRAN_ID TAOS_DEF_ERROR_CODE(0, 0x000F) +#define TSDB_CODE_RPC_INVALID_SESSION_ID TAOS_DEF_ERROR_CODE(0, 0x0010) +#define TSDB_CODE_RPC_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x0011) +#define TSDB_CODE_RPC_INVALID_RESPONSE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0012) +#define TSDB_CODE_RPC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0013) +#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) +#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) +#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) //common & util #define TSDB_CODE_OPS_NOT_SUPPORT TAOS_DEF_ERROR_CODE(0, 0x0100) @@ -73,56 +73,56 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) - +#define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) +#define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) #define TSDB_CODE_REF_INVALID_ID TAOS_DEF_ERROR_CODE(0, 0x0113) #define TSDB_CODE_REF_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0114) #define TSDB_CODE_REF_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0115) - #define TSDB_CODE_INVALID_VERSION_NUMBER TAOS_DEF_ERROR_CODE(0, 0x0120) #define TSDB_CODE_INVALID_VERSION_STRING TAOS_DEF_ERROR_CODE(0, 0x0121) #define TSDB_CODE_VERSION_NOT_COMPATIBLE TAOS_DEF_ERROR_CODE(0, 0x0122) //client -#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) //"Invalid Operation") -#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) //"Invalid qhandle") -#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) //"Invalid combination of client/service time") -#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) //"Invalid value in client") -#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) //"Invalid client version") -#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) //"Invalid client ie") -#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) //"Invalid host name") -#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) //"Invalid user name") -#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) //"Invalid password") -#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) //"Database name too long") -#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) //"Table name too long") -#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) //"Invalid connection") -#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) //"System out of memory") -#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) //"System out of disk space") -#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) //"Query cache erased") -#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) //"Query terminated") -#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) //"Result set too large to be sorted") // too many result for ordered super table projection query -#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) //"Application error") -#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) //"Action in progress") -#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) //"Disconnected from service") -#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) //"No write permission") -#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) //"Connection killed") -#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) //"Syntax error in SQL") -#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available") -#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist") -#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config") -#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty") -#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line") -#define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached") -#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names") -#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length") -#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length") -#define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) //"duplicated tag names") -#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) //"Invalid JSON format") -#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) //"Invalid JSON data type") -#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) //"Value out of range") -#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) //"Invalid tsc input") +#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) +#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) +#define TSDB_CODE_TSC_INVALID_TIME_STAMP TAOS_DEF_ERROR_CODE(0, 0x0202) +#define TSDB_CODE_TSC_INVALID_VALUE TAOS_DEF_ERROR_CODE(0, 0x0203) +#define TSDB_CODE_TSC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0204) +#define TSDB_CODE_TSC_INVALID_IE TAOS_DEF_ERROR_CODE(0, 0x0205) +#define TSDB_CODE_TSC_INVALID_FQDN TAOS_DEF_ERROR_CODE(0, 0x0206) +#define TSDB_CODE_TSC_INVALID_USER_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0207) +#define TSDB_CODE_TSC_INVALID_PASS_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0208) +#define TSDB_CODE_TSC_INVALID_DB_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0209) +#define TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH TAOS_DEF_ERROR_CODE(0, 0x020A) +#define TSDB_CODE_TSC_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x020B) +#define TSDB_CODE_TSC_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x020C) +#define TSDB_CODE_TSC_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x020D) +#define TSDB_CODE_TSC_QUERY_CACHE_ERASED TAOS_DEF_ERROR_CODE(0, 0x020E) +#define TSDB_CODE_TSC_QUERY_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x020F) +#define TSDB_CODE_TSC_SORTED_RES_TOO_MANY TAOS_DEF_ERROR_CODE(0, 0x0210) +#define TSDB_CODE_TSC_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0211) +#define TSDB_CODE_TSC_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0212) +#define TSDB_CODE_TSC_DISCONNECTED TAOS_DEF_ERROR_CODE(0, 0x0213) +#define TSDB_CODE_TSC_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0214) +#define TSDB_CODE_TSC_CONN_KILLED TAOS_DEF_ERROR_CODE(0, 0x0215) +#define TSDB_CODE_TSC_SQL_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x0216) +#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) +#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) +#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) +#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) +#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) +#define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) +#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) +#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) +#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) +#define TSDB_CODE_TSC_DUP_TAG_NAMES TAOS_DEF_ERROR_CODE(0, 0x0220) +#define TSDB_CODE_TSC_INVALID_JSON TAOS_DEF_ERROR_CODE(0, 0x0221) +#define TSDB_CODE_TSC_INVALID_JSON_TYPE TAOS_DEF_ERROR_CODE(0, 0x0222) +#define TSDB_CODE_TSC_VALUE_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x0223) +#define TSDB_CODE_TSC_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0X0224) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -235,6 +235,9 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AD) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) +// mnode-infoSchema +#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) + // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) #define TSDB_CODE_MND_FUNC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C1) @@ -265,6 +268,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_MQ_PLACEHOLDER TAOS_DEF_ERROR_CODE(0, 0x03F0) + + // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) #define TSDB_CODE_DND_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0401) @@ -299,174 +304,174 @@ int32_t* taosGetErrno(); #define TSDB_CODE_DND_VNODE_TOO_MANY_VNODES TAOS_DEF_ERROR_CODE(0, 0x0465) // vnode -#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) //"Action in progress") -#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) //"Message not processed") -#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) //"Action need to be reprocessed") -#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) //"Invalid Vgroup ID") -#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) //"Vnode initialization failed") -#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) //"System out of disk space") -#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) //"No write permission for disk files") -#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) //"Missing data file") -#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) //"Out of memory") -#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) //"Unexpected generic error in vnode") -#define TSDB_CODE_VND_INVALID_CFG_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) //"Invalid config file) -#define TSDB_CODE_VND_INVALID_TERM_FILE TAOS_DEF_ERROR_CODE(0, 0x050B) //"Invalid term file") -#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) //"Database memory is full") -#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) //"Database is dropping") -#define TSDB_CODE_VND_IS_UPDATING TAOS_DEF_ERROR_CODE(0, 0x050E) //"Database is updating") -#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) //"Database is closing") -#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) //"Database suspended") -#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) //"Database write operation denied") -#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) //"Database is syncing") -#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) //"Invalid tsdb state") -#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // "Table not exists") +#define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) +#define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) +#define TSDB_CODE_VND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0502) +#define TSDB_CODE_VND_INVALID_VGROUP_ID TAOS_DEF_ERROR_CODE(0, 0x0503) +#define TSDB_CODE_VND_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0504) +#define TSDB_CODE_VND_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0505) +#define TSDB_CODE_VND_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0506) +#define TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR TAOS_DEF_ERROR_CODE(0, 0x0507) +#define TSDB_CODE_VND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0508) +#define TSDB_CODE_VND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0509) +#define TSDB_CODE_VND_INVALID_CFG_FILE TAOS_DEF_ERROR_CODE(0, 0x050A) +#define TSDB_CODE_VND_INVALID_TERM_FILE TAOS_DEF_ERROR_CODE(0, 0x050B) +#define TSDB_CODE_VND_IS_FLOWCTRL TAOS_DEF_ERROR_CODE(0, 0x050C) +#define TSDB_CODE_VND_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x050D) +#define TSDB_CODE_VND_IS_UPDATING TAOS_DEF_ERROR_CODE(0, 0x050E) +#define TSDB_CODE_VND_IS_CLOSING TAOS_DEF_ERROR_CODE(0, 0x0510) +#define TSDB_CODE_VND_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0511) +#define TSDB_CODE_VND_NO_WRITE_AUTH TAOS_DEF_ERROR_CODE(0, 0x0512) +#define TSDB_CODE_VND_IS_SYNCING TAOS_DEF_ERROR_CODE(0, 0x0513) +#define TSDB_CODE_VND_INVALID_TSDB_STATE TAOS_DEF_ERROR_CODE(0, 0x0514) +#define TSDB_CODE_VND_TB_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0515) // tsdb -#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) //"Invalid table ID") -#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) //"Invalid table type") -#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) //"Invalid table schema version") -#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) //"Table already exists") -#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) //"Invalid configuration") -#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) //"Tsdb init failed") -#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) //"No diskspace for tsdb") -#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) //"No permission for disk files") -#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) //"Data file(s) corrupted") -#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) //"Out of memory") -#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) //"Tag too old") -#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) //"Timestamp data out of range") -#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) //"Submit message is messed up") -#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) //"Invalid operation") -#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) //"Invalid creation of table") -#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) //"No table data in memory skiplist") -#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) //"File already exists") -#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) //"Need to reconfigure table") -#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") -#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") -#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") -#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") -#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) //"TSDB no cache last row data") +#define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) +#define TSDB_CODE_TDB_INVALID_TABLE_TYPE TAOS_DEF_ERROR_CODE(0, 0x0601) +#define TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION TAOS_DEF_ERROR_CODE(0, 0x0602) +#define TSDB_CODE_TDB_TABLE_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0603) +#define TSDB_CODE_TDB_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0604) +#define TSDB_CODE_TDB_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0605) +#define TSDB_CODE_TDB_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0606) +#define TSDB_CODE_TDB_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0607) +#define TSDB_CODE_TDB_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0608) +#define TSDB_CODE_TDB_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0609) +#define TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE TAOS_DEF_ERROR_CODE(0, 0x060A) +#define TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE TAOS_DEF_ERROR_CODE(0, 0x060B) +#define TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x060C) +#define TSDB_CODE_TDB_INVALID_ACTION TAOS_DEF_ERROR_CODE(0, 0x060D) +#define TSDB_CODE_TDB_INVALID_CREATE_TB_MSG TAOS_DEF_ERROR_CODE(0, 0x060E) +#define TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM TAOS_DEF_ERROR_CODE(0, 0x060F) +#define TSDB_CODE_TDB_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0610) +#define TSDB_CODE_TDB_TABLE_RECONFIGURE TAOS_DEF_ERROR_CODE(0, 0x0611) +#define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) +#define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) +#define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) +#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) // query -#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") -#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) //"Invalid message") // failed to validate the sql expression msg by vnode -#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) //"No diskspace for query") -#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) //"System out of memory") -#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) //"Unexpected generic error in query") -#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) //"Duplicated join key") -#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) //"Tag conditon too many") -#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) //"Query not ready") -#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) //"Query should response") -#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) //"Multiple retrieval of this query") -#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query") -#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached") -#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") -#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition") -#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error") -#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) //"invalid input") -#define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) //"Scheduler not exist") -#define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) //"Task not exist") -#define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) //"Task already exist") -#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) //"Task context not exist") -#define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) //"Task cancelled") -#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) //"Task dropped") -#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) //"Task cancelling") -#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) //"Task dropping") -#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) //"Duplicatted operation") -#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) //"Task message error") -#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) //"Job freed") -#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) //"Task status error") +#define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) +#define TSDB_CODE_QRY_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0701) +#define TSDB_CODE_QRY_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0702) +#define TSDB_CODE_QRY_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0703) +#define TSDB_CODE_QRY_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0704) +#define TSDB_CODE_QRY_DUP_JOIN_KEY TAOS_DEF_ERROR_CODE(0, 0x0705) +#define TSDB_CODE_QRY_EXCEED_TAGS_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0706) +#define TSDB_CODE_QRY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0707) +#define TSDB_CODE_QRY_HAS_RSP TAOS_DEF_ERROR_CODE(0, 0x0708) +#define TSDB_CODE_QRY_IN_EXEC TAOS_DEF_ERROR_CODE(0, 0x0709) +#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) +#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) +#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) +#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) +#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) +#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) +#define TSDB_CODE_QRY_SCH_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0710) +#define TSDB_CODE_QRY_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0711) +#define TSDB_CODE_QRY_TASK_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0712) +#define TSDB_CODE_QRY_TASK_CTX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0713) +#define TSDB_CODE_QRY_TASK_CANCELLED TAOS_DEF_ERROR_CODE(0, 0x0714) +#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0715) +#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0716) +#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0717) +#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0718) +#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719) +#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A) +#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B) // grant -#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) //"License expired") -#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) //"DNode creation limited by licence") -#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) //"Account creation limited by license") -#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) //"Table creation limited by license") -#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) //"DB creation limited by license") -#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) //"User creation limited by license") -#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) //"Conn creation limited by license") -#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) //"Stream creation limited by license") -#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) //"Write speed limited by license") -#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) //"Storage capacity limited by license") -#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) //"Query time limited by license") -#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) //"CPU cores limited by license") +#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) +#define TSDB_CODE_GRANT_DNODE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0801) +#define TSDB_CODE_GRANT_ACCT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0802) +#define TSDB_CODE_GRANT_TIMESERIES_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0803) +#define TSDB_CODE_GRANT_DB_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0804) +#define TSDB_CODE_GRANT_USER_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0805) +#define TSDB_CODE_GRANT_CONN_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0806) +#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807) +#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808) +#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809) +#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A) +#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B) // sync -#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) //"Invalid Sync Configuration") -#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) //"Sync module not enabled") -#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) //"Invalid Sync version") -#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) //"Sync confirm expired") -#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) //"Too many sync fwd infos") -#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) //"Mismatched protocol") -#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) //"Mismatched clusterId") -#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) //"Mismatched signature") -#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) //"Invalid msg checksum") -#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") -#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") +#define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) +#define TSDB_CODE_SYN_NOT_ENABLED TAOS_DEF_ERROR_CODE(0, 0x0901) +#define TSDB_CODE_SYN_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0902) +#define TSDB_CODE_SYN_CONFIRM_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0903) +#define TSDB_CODE_SYN_TOO_MANY_FWDINFO TAOS_DEF_ERROR_CODE(0, 0x0904) +#define TSDB_CODE_SYN_MISMATCHED_PROTOCOL TAOS_DEF_ERROR_CODE(0, 0x0905) +#define TSDB_CODE_SYN_MISMATCHED_CLUSTERID TAOS_DEF_ERROR_CODE(0, 0x0906) +#define TSDB_CODE_SYN_MISMATCHED_SIGNATURE TAOS_DEF_ERROR_CODE(0, 0x0907) +#define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) +#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) +#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // tq -#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration") -#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed") -#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq") -#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files") -#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted") -#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory") -#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists") -#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir") -#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found") -#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction") -#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction") -#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode") +#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) +#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) +#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) +#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) +#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) +#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) +#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) +#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) +#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) +#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) +#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) +#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) // wal -#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") -#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") -#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") -#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) //"WAL invalid version") -#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory") +#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) +#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) +#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) +#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) +#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // tfs -#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") -#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) //"tfs invalid mount config") -#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) //"tfs too many mount") -#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) //"tfs duplicate primary mount") -#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) //"tfs no primary mount") -#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) //"tfs no mount at tier") -#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) //"tfs file already exists") -#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) //"tfs invalid level") -#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) //"tfs no valid disk") +#define TSDB_CODE_FS_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x2200) +#define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) +#define TSDB_CODE_FS_TOO_MANY_MOUNT TAOS_DEF_ERROR_CODE(0, 0x2202) +#define TSDB_CODE_FS_DUP_PRIMARY TAOS_DEF_ERROR_CODE(0, 0x2203) +#define TSDB_CODE_FS_NO_PRIMARY_DISK TAOS_DEF_ERROR_CODE(0, 0x2204) +#define TSDB_CODE_FS_NO_MOUNT_AT_TIER TAOS_DEF_ERROR_CODE(0, 0x2205) +#define TSDB_CODE_FS_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2206) +#define TSDB_CODE_FS_INVLD_LEVEL TAOS_DEF_ERROR_CODE(0, 0x2207) +#define TSDB_CODE_FS_NO_VALID_DISK TAOS_DEF_ERROR_CODE(0, 0x2208) // monitor -#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) //"monitor invalid monitor db connection") +#define TSDB_CODE_MON_CONNECTION_INVALID TAOS_DEF_ERROR_CODE(0, 0x2300) // catalog -#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) //catalog interval error -#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) //invalid catalog input parameters -#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) //catalog is not ready -#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) //catalog memory error -#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) //catalog system error -#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) //Database is dropped -#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) //catalog is out of service +#define TSDB_CODE_CTG_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2400) +#define TSDB_CODE_CTG_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2401) +#define TSDB_CODE_CTG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x2402) +#define TSDB_CODE_CTG_MEM_ERROR TAOS_DEF_ERROR_CODE(0, 0x2403) +#define TSDB_CODE_CTG_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2404) +#define TSDB_CODE_CTG_DB_DROPPED TAOS_DEF_ERROR_CODE(0, 0x2405) +#define TSDB_CODE_CTG_OUT_OF_SERVICE TAOS_DEF_ERROR_CODE(0, 0x2406) //scheduler -#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error -#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error +#define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) +#define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //parser -#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) //Invalid column name -#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) //Table does not exist -#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) //Column ambiguously defined -#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) //Invalid value type -#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605) //Invalid function name -#define TSDB_CODE_PAR_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2606) //Invalid number of arguments -#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) //Inconsistent datatypes -#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) //There mustn't be aggregation -#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) //ORDER BY item must be the number of a SELECT-list expression -#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression -#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression -#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function +#define TSDB_CODE_PAR_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) +#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) +#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) +#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) +#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605) +#define TSDB_CODE_PAR_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2606) +#define TSDB_CODE_PAR_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2607) +#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) +#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) +#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) +#define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) +#define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TAOS_ERROR_H_*/ +#endif /*_TD_UTIL_TAOS_ERROR_H_*/ diff --git a/include/util/tarray.h b/include/util/tarray.h index 6d6120a49b5c7be84cf97a2af0cc7f44e6298966..521e54040d6abd8ac53af11d830325bc3b5c053c 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -13,16 +13,15 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_ARRAY_H -#define _TD_UTIL_ARRAY_H +#ifndef _TD_UTIL_ARRAY_H_ +#define _TD_UTIL_ARRAY_H_ + +#include "talgo.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "talgo.h" - #if 0 #define TARRAY(TYPE) \ struct { \ @@ -31,20 +30,20 @@ extern "C" { struct TYPE* td_array_data_; \ } -#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ -#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ +#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ +#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ #define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) #endif -#define TARRAY_MIN_SIZE 8 +#define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) -#define TARRAY_GET_START(array) ((array)->pData) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; - size_t capacity; - size_t elemSize; + uint32_t capacity; + uint32_t elemSize; void* pData; } SArray; @@ -70,7 +69,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); * @param nEles * @return */ -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles); +void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles); /** * @@ -238,7 +237,7 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn); * @param compar * @param key */ -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags); /** * search the array, return index of the element @@ -246,14 +245,14 @@ void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t compa * @param compar * @param key */ -int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags); +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags); /** * search the array * @param pArray * @param key */ -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags); /** * sort the pointer data in the array @@ -269,4 +268,4 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par } #endif -#endif /*_TD_UTIL_ARRAY_H*/ +#endif /*_TD_UTIL_ARRAY_H_*/ diff --git a/include/util/tbase64.h b/include/util/tbase64.h new file mode 100644 index 0000000000000000000000000000000000000000..22b8f95c5a8868ccd508841df091674d0dab6a76 --- /dev/null +++ b/include/util/tbase64.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_BASE64_H_ +#define _TD_UTIL_BASE64_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen); +char *base64_encode(const uint8_t *value, int32_t vlen); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_BASE64_H_*/ \ No newline at end of file diff --git a/include/util/tbuffer.h b/include/util/tbuffer.h index 6bb7f67e3d52549bac2a381bc4d65ad1b6263bf5..2ed1b7326bdb27ffe182b30d67a9369680898d23 100644 --- a/include/util/tbuffer.h +++ b/include/util/tbuffer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_BUFFER_H -#define _TD_UTIL_BUFFER_H +#ifndef _TD_UTIL_BUFFER_H_ +#define _TD_UTIL_BUFFER_H_ #include "os.h" @@ -26,9 +26,9 @@ extern "C" { // usage example /* #include -#include "exception.h" +#include "texception.h" -int main( int argc, char** argv ) { +int32_t main( int32_t argc, char** argv ) { SBufferWriter bw = tbufInitWriter( NULL, false ); TRY( 1 ) { @@ -39,7 +39,7 @@ int main( int argc, char** argv ) { // reserve space for the interger count size_t pos = tbufReserve( &bw, sizeof(int32_t) ); // write 5 integers to the buffer - for( int i = 0; i < 5; i++) { + for( int32_t i = 0; i < 5; i++) { tbufWriteInt32( &bw, i ); } // write the integer count to buffer at reserved position @@ -55,7 +55,7 @@ int main( int argc, char** argv ) { SBufferReader br = tbufInitReader( data, size, false ); // read & print out all integers int32_t count = tbufReadInt32( &br ); - for( int i = 0; i < count; i++ ) { + for( int32_t i = 0; i < count; i++ ) { printf( "%d\n", tbufReadInt32(&br) ); } // read & print out a string @@ -87,12 +87,10 @@ typedef struct SBufferWriter { void* (*allocator)(void*, size_t); } SBufferWriter; -//////////////////////////////////////////////////////////////////////////////// // common functions & macros for both reader & writer #define tbufTell(buf) ((buf)->pos) -//////////////////////////////////////////////////////////////////////////////// /* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */ // *Allocator*, function to allocate memory, will use 'realloc' if NULL // *Endian*, if true, writer functions of primitive types will do 'hton' automatically @@ -167,4 +165,4 @@ double tbufReadDouble(SBufferReader* buf); } #endif -#endif /*_TD_UTIL_BUFFER_H*/ +#endif /*_TD_UTIL_BUFFER_H_*/ diff --git a/include/util/tcache.h b/include/util/tcache.h index 11f9044d2838e8f73e9fcb01f6dc554b1de6aeff..0de3ab3a2801754f35f894ab3e3a02275cc5020b 100644 --- a/include/util/tcache.h +++ b/include/util/tcache.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CACHE_H -#define _TD_UTIL_CACHE_H +#ifndef _TD_UTIL_CACHE_H_ +#define _TD_UTIL_CACHE_H_ + +#include "thash.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tlockfree.h" -#include "thash.h" - -#if defined(_TD_ARM_32) - #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT - #define TSDB_CACHE_PTR_TYPE int32_t +#if defined(_TD_ARM_32) +#define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_INT +#define TSDB_CACHE_PTR_TYPE int32_t #else - #define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT - #define TSDB_CACHE_PTR_TYPE int64_t +#define TSDB_CACHE_PTR_KEY TSDB_DATA_TYPE_BIGINT +#define TSDB_CACHE_PTR_TYPE int64_t #endif -typedef void (*__cache_free_fn_t)(void*); -typedef void (*__cache_trav_fn_t)(void*, void*); +typedef void (*__cache_free_fn_t)(void *); +typedef void (*__cache_trav_fn_t)(void *, void *); typedef struct SCacheStatis { int64_t missCount; @@ -45,17 +43,17 @@ typedef struct SCacheStatis { struct STrashElem; typedef struct SCacheDataNode { - uint64_t addedTime; // the added time when this element is added or updated into cache - uint64_t lifespan; // life duration when this element should be remove from cache - uint64_t expireTime; // expire time + uint64_t addedTime; // the added time when this element is added or updated into cache + uint64_t lifespan; // life duration when this element should be remove from cache + uint64_t expireTime; // expire time uint64_t signature; - struct STrashElem *pTNodeHeader; // point to trash node head - uint16_t keySize: 15; // max key size: 32kb - bool inTrashcan: 1;// denote if it is in trash or not - uint32_t size; // allocated size for current SCacheDataNode + struct STrashElem *pTNodeHeader; // point to trash node head + uint16_t keySize : 15; // max key size: 32kb + bool inTrashcan : 1; // denote if it is in trash or not + uint32_t size; // allocated size for current SCacheDataNode T_REF_DECLARE() - char *key; - char data[]; + char *key; + char data[]; } SCacheDataNode; typedef struct STrashElem { @@ -73,22 +71,22 @@ typedef struct STrashElem { * when the node in pTrash does not be referenced, it will be release at the expired expiredTime */ typedef struct { - int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. - int64_t refreshTime; - STrashElem * pTrash; - char* name; - SCacheStatis statistics; - SHashObj * pHashTable; + int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included. + int64_t refreshTime; + STrashElem *pTrash; + char *name; + SCacheStatis statistics; + SHashObj *pHashTable; __cache_free_fn_t freeFp; - uint32_t numOfElemsInTrash; // number of element in trash - uint8_t deleting; // set the deleting flag to stop refreshing ASAP. - pthread_t refreshWorker; - bool extendLifespan; // auto extend life span when one item is accessed. - int64_t checkTick; // tick used to record the check times of the refresh threads + uint32_t numOfElemsInTrash; // number of element in trash + uint8_t deleting; // set the deleting flag to stop refreshing ASAP. + pthread_t refreshWorker; + bool extendLifespan; // auto extend life span when one item is accessed. + int64_t checkTick; // tick used to record the check times of the refresh threads #if defined(LINUX) pthread_rwlock_t lock; #else - pthread_mutex_t lock; + pthread_mutex_t lock; #endif } SCacheObj; @@ -101,7 +99,8 @@ typedef struct { * @param fn free resource callback function * @return */ -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char *cacheName); +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, + const char *cacheName); /** * add data into cache @@ -113,7 +112,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, + int32_t durationMS); /** * get data from cache @@ -177,7 +177,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj); * @param fp * @return */ -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1); +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1); /** * stop background refresh worker thread @@ -188,4 +188,4 @@ void taosStopCacheRefreshWorker(); } #endif -#endif /*_TD_UTIL_CACHE_H*/ +#endif /*_TD_UTIL_CACHE_H_*/ diff --git a/include/util/tchecksum.h b/include/util/tchecksum.h index fb6c85edcda5e70de8ba9ad18e8dfd1b9317e4ed..28fb784c467bd24fe92be2719cb6f983f08e0181 100644 --- a/include/util/tchecksum.h +++ b/include/util/tchecksum.h @@ -13,24 +13,22 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CHECKSUM_H -#define _TD_UTIL_CHECKSUM_H +#ifndef _TD_UTIL_CHECKSUM_H_ +#define _TD_UTIL_CHECKSUM_H_ + +#include "tcrc32c.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tcrc32c.h" -#include "tutil.h" - typedef uint32_t TSCKSUM; static FORCE_INLINE TSCKSUM taosCalcChecksum(TSCKSUM csi, const uint8_t *stream, uint32_t ssize) { return (*crc32c)(csi, stream, (size_t)ssize); } -static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uint32_t ssize) { +static FORCE_INLINE int32_t taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uint32_t ssize) { if (ssize < sizeof(TSCKSUM)) return -1; *((TSCKSUM *)(stream + ssize - sizeof(TSCKSUM))) = (*crc32c)(csi, stream, (size_t)(ssize - sizeof(TSCKSUM))); @@ -38,11 +36,11 @@ static FORCE_INLINE int taosCalcChecksumAppend(TSCKSUM csi, uint8_t *stream, uin return 0; } -static FORCE_INLINE int taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { +static FORCE_INLINE int32_t taosCheckChecksum(const uint8_t *stream, uint32_t ssize, TSCKSUM checksum) { return (checksum != (*crc32c)(0, stream, (size_t)ssize)); } -static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { +static FORCE_INLINE int32_t taosCheckChecksumWhole(const uint8_t *stream, uint32_t ssize) { if (ssize < sizeof(TSCKSUM)) return 0; #if (_WIN64) @@ -56,4 +54,4 @@ static FORCE_INLINE int taosCheckChecksumWhole(const uint8_t *stream, uint32_t s } #endif -#endif /*_TD_UTIL_CHECKSUM_H*/ +#endif /*_TD_UTIL_CHECKSUM_H_*/ diff --git a/include/util/tcoding.h b/include/util/tcoding.h index fed9d12cee62a01a2b92215326f4465beb760a1a..943a1f9ecad450f33b4f0e0eb9ef28fae60bd813 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -12,29 +12,30 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_CODING_H -#define _TD_UTIL_CODING_H + +#ifndef _TD_UTIL_CODING_H_ +#define _TD_UTIL_CODING_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -#define ENCODE_LIMIT (((uint8_t)1) << 7) +#define ENCODE_LIMIT (((uint8_t)1) << 7) #define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode /* ------------------------ LEGACY CODES ------------------------ */ #if 1 // ---- Fixed U8 -static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU8(void **buf, uint8_t value) { if (buf != NULL) { ((uint8_t *)(*buf))[0] = value; *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { @@ -43,12 +44,12 @@ static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { } // ---- Fixed I8 -static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI8(void **buf, int8_t value) { if (buf != NULL) { ((int8_t *)(*buf))[0] = value; *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { @@ -57,7 +58,7 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { } // ---- Fixed U16 -static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -68,7 +69,7 @@ static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { @@ -83,7 +84,7 @@ static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { } // ---- Fixed I16 -static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } @@ -95,7 +96,7 @@ static FORCE_INLINE void *taosDecodeFixedI16(const void *buf, int16_t *value) { } // ---- Fixed U32 -static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU32(void **buf, uint32_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -108,7 +109,7 @@ static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { @@ -125,7 +126,7 @@ static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { } // ---- Fixed I32 -static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } @@ -137,7 +138,7 @@ static FORCE_INLINE void *taosDecodeFixedI32(const void *buf, int32_t *value) { } // ---- Fixed U64 -static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { +static FORCE_INLINE int32_t taosEncodeFixedU64(void **buf, uint64_t value) { if (buf != NULL) { if (IS_LITTLE_ENDIAN()) { memcpy(*buf, &value, sizeof(value)); @@ -155,7 +156,7 @@ static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return (int)sizeof(value); + return (int32_t)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { @@ -176,7 +177,7 @@ static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { } // ---- Fixed I64 -static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { +static FORCE_INLINE int32_t taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } @@ -188,8 +189,8 @@ static FORCE_INLINE void *taosDecodeFixedI64(const void *buf, int64_t *value) { } // ---- Variant U16 -static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU16(void **buf, uint16_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; @@ -206,7 +207,7 @@ static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { } static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) { - int i = 0; + int32_t i = 0; uint16_t tval = 0; *value = 0; while (i < 3) { @@ -224,7 +225,7 @@ static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) } // ---- Variant I16 -static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } @@ -236,8 +237,8 @@ static FORCE_INLINE void *taosDecodeVariantI16(const void *buf, int16_t *value) } // ---- Variant U32 -static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU32(void **buf, uint32_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; @@ -254,7 +255,7 @@ static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { } static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) { - int i = 0; + int32_t i = 0; uint32_t tval = 0; *value = 0; while (i < 5) { @@ -272,7 +273,7 @@ static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) } // ---- Variant I32 -static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } @@ -284,8 +285,8 @@ static FORCE_INLINE void *taosDecodeVariantI32(const void *buf, int32_t *value) } // ---- Variant U64 -static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { - int i = 0; +static FORCE_INLINE int32_t taosEncodeVariantU64(void **buf, uint64_t value) { + int32_t i = 0; while (value >= ENCODE_LIMIT) { if (buf != NULL) ((uint8_t *)(*buf))[i] = (uint8_t)(value | ENCODE_LIMIT); value >>= 7; @@ -302,7 +303,7 @@ static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { } static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) { - int i = 0; + int32_t i = 0; uint64_t tval = 0; *value = 0; while (i < 10) { @@ -320,7 +321,7 @@ static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) } // ---- Variant I64 -static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { +static FORCE_INLINE int32_t taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } @@ -332,16 +333,16 @@ static FORCE_INLINE void *taosDecodeVariantI64(const void *buf, int64_t *value) } // ---- string -static FORCE_INLINE int taosEncodeString(void **buf, const char *value) { - int tlen = 0; - size_t size = strlen(value); +static FORCE_INLINE int32_t taosEncodeString(void **buf, const char *value) { + int32_t tlen = 0; + size_t size = strlen(value); tlen += taosEncodeVariantU64(buf, size); if (buf != NULL) { memcpy(*buf, value, size); *buf = POINTER_SHIFT(*buf, size); } - tlen += (int)size; + tlen += (int32_t)size; return tlen; } @@ -372,14 +373,14 @@ static FORCE_INLINE void *taosDecodeStringTo(const void *buf, char *value) { } // ---- binary -static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { - int tlen = 0; +static FORCE_INLINE int32_t taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { + int32_t tlen = 0; if (buf != NULL) { memcpy(*buf, value, valueLen); *buf = POINTER_SHIFT(*buf, valueLen); } - tlen += (int)valueLen; + tlen += (int32_t)valueLen; return tlen; } @@ -403,4 +404,4 @@ static FORCE_INLINE void *taosDecodeBinaryTo(const void *buf, void *value, int32 } #endif -#endif /*_TD_UTIL_CODING_H*/ +#endif /*_TD_UTIL_CODING_H_*/ diff --git a/include/util/compare.h b/include/util/tcompare.h similarity index 87% rename from include/util/compare.h rename to include/util/tcompare.h index 6305334e02444c3b6fae8d455cb7c5c241a1df84..4c80eeb4f601034852acebd52b2e18b74c52d424 100644 --- a/include/util/compare.h +++ b/include/util/tcompare.h @@ -16,17 +16,18 @@ #ifndef _TD_UTIL_COMPARE_H_ #define _TD_UTIL_COMPARE_H_ +#include "os.h" +#include "taos.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -#define TSDB_PATTERN_MATCH 0 -#define TSDB_PATTERN_NOMATCH 1 -#define TSDB_PATTERN_NOWILDCARDMATCH 2 +#define TSDB_PATTERN_MATCH 0 +#define TSDB_PATTERN_NOMATCH 1 +#define TSDB_PATTERN_NOWILDCARDMATCH 2 #define TSDB_PATTERN_STRING_DEFAULT_LEN 100 -#define TSDB_REGEX_STRING_DEFAULT_LEN 128 +#define TSDB_REGEX_STRING_DEFAULT_LEN 128 #define FLT_COMPAR_TOL_FACTOR 4 #define FLT_EQUAL(_x, _y) (fabs((_x) - (_y)) <= (FLT_COMPAR_TOL_FACTOR * FLT_EPSILON)) @@ -62,7 +63,6 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight); int32_t compareChkInString(const void *pLeft, const void *pRight); int32_t compareChkNotInString(const void *pLeft, const void *pRight); - int32_t compareInt8Val(const void *pLeft, const void *pRight); int32_t compareInt16Val(const void *pLeft, const void *pRight); int32_t compareInt32Val(const void *pLeft, const void *pRight); @@ -83,7 +83,6 @@ int32_t compareStrRegexComp(const void *pLeft, const void *pRight); int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight); int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight); - int32_t compareInt8ValDesc(const void *pLeft, const void *pRight); int32_t compareInt16ValDesc(const void *pLeft, const void *pRight); int32_t compareInt32ValDesc(const void *pLeft, const void *pRight); @@ -100,8 +99,15 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight); int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight); -__compar_fn_t getComparFunc(int32_t type, int32_t optr); +int32_t compareStrPatternMatch(const void *pLeft, const void *pRight); +int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight); +int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight); +int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight); + +__compar_fn_t getComparFunc(int32_t type, int32_t optr); +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); +int32_t doCompare(const char *a, const char *b, int32_t type, size_t size); #ifdef __cplusplus } diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 4cb11935783504ba2792394baaf3942a560a7279..d7ba4dfa3fd9b255ded7cfe58cd2ad859575d445 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -13,22 +13,23 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_COMPRESSION_H -#define _TD_UTIL_COMPRESSION_H +#ifndef _TD_UTIL_COMPRESSION_H_ +#define _TD_UTIL_COMPRESSION_H_ + +#include "os.h" +#include "taos.h" +#include "tutil.h" #ifdef __cplusplus extern "C" { #endif -#include "taos.h" -#include "tutil.h" - #define COMP_OVERFLOW_BYTES 2 -#define BITS_PER_BYTE 8 +#define BITS_PER_BYTE 8 // Masks #define INT64MASK(_x) ((((uint64_t)1) << _x) - 1) #define INT32MASK(_x) (((uint32_t)1 << _x) - 1) -#define INT8MASK(_x) (((uint8_t)1 << _x) - 1) +#define INT8MASK(_x) (((uint8_t)1 << _x) - 1) // Compression algorithm #define NO_COMPRESSION 0 #define ONE_STAGE_COMP 1 @@ -41,48 +42,51 @@ extern "C" { // // compression data mode save first byte lower 1 bit -#define MODE_NOCOMPRESS 0 // original data -#define MODE_COMPRESS 1 // compatible old compress +#define MODE_NOCOMPRESS 0 // original data +#define MODE_COMPRESS 1 // compatible old compress // compression algorithm save first byte higher 7 bit -#define ALGO_SZ_LOSSY 1 // SZ compress - -#define HEAD_MODE(x) x%2 -#define HEAD_ALGO(x) x/2 - -extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type); -extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type); -extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output); -extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize); -extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize); -extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output); -extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output); -extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output); -extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output); +#define ALGO_SZ_LOSSY 1 // SZ compress + +#define HEAD_MODE(x) x % 2 +#define HEAD_ALGO(x) x / 2 + +extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type); +extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, + const char type); +extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize); +extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, + int32_t outputSize); +extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); +extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); // lossy -extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); -extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); -extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); -extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); +extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output); +extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output); +extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output); +extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output); #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; -// init call -int tsCompressInit(); -// exit call -void tsCompressExit(); +int32_t tsCompressInit(); +void tsCompressExit(); #endif -static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -90,8 +94,9 @@ static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize } } -static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -103,12 +108,13 @@ static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compres } } -static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -116,8 +122,9 @@ static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSiz } } -static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -129,12 +136,13 @@ static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compre } } -static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, - char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -142,8 +150,9 @@ static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, co } } -static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); } else if (algorithm == TWO_STAGE_COMP) { @@ -155,12 +164,13 @@ static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedS } } -static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -168,8 +178,9 @@ static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, } } -static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); } else if (algorithm == TWO_STAGE_COMP) { @@ -181,12 +192,13 @@ static FORCE_INLINE int tsDecompressBigint(const char *const input, int compress } } -static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressBoolImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressBoolImp(input, nelements, buffer); + int32_t len = tsCompressBoolImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -194,8 +206,9 @@ static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, c } } -static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressBoolImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { @@ -207,47 +220,51 @@ static FORCE_INLINE int tsDecompressBool(const char *const input, int compressed } } -static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { return tsCompressStringImp(input, inputSize, output, outputSize); } -static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsDecompressStringImp(input, compressedSize, output, outputSize); } -static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { #ifdef TD_TSZ // lossy mode - if(lossyFloat) { + if (lossyFloat) { return tsCompressFloatLossyImp(input, nelements, output); - // lossless mode + // lossless mode } else { -#endif +#endif if (algorithm == ONE_STAGE_COMP) { return tsCompressFloatImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressFloatImp(input, nelements, buffer); + int32_t len = tsCompressFloatImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); return -1; - } -#ifdef TD_TSZ + } +#ifdef TD_TSZ } #endif } -static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { #ifdef TD_TSZ - if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); } else { -#endif +#endif // decompress lossless if (algorithm == ONE_STAGE_COMP) { return tsDecompressFloatImp(input, nelements, output); @@ -258,43 +275,44 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } #endif } - -static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { -#ifdef TD_TSZ - if(lossyDouble){ +static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { +#ifdef TD_TSZ + if (lossyDouble) { // lossy mode return tsCompressDoubleLossyImp(input, nelements, output); } else { -#endif +#endif // lossless mode if (algorithm == ONE_STAGE_COMP) { return tsCompressDoubleImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressDoubleImp(input, nelements, buffer); + int32_t len = tsCompressDoubleImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } -#endif +#endif } -static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { - #ifdef TD_TSZ - if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ +static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } else { - #endif +#endif // decompress lossless if (algorithm == ONE_STAGE_COMP) { return tsDecompressDoubleImp(input, nelements, output); @@ -305,43 +323,48 @@ static FORCE_INLINE int tsDecompressDouble(const char *const input, int compress assert(0); return -1; } -#ifdef TD_TSZ +#ifdef TD_TSZ } -#endif +#endif } -#ifdef TD_TSZ +#ifdef TD_TSZ // // lossy float double // -static FORCE_INLINE int tsCompressFloatLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsCompressFloatLossyImp(input, nelements, output); } -static FORCE_INLINE int tsDecompressFloatLossy(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsDecompressFloatLossy(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); } -static FORCE_INLINE int tsCompressDoubleLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsCompressDoubleLossy(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { return tsCompressDoubleLossyImp(input, nelements, output); } -static FORCE_INLINE int tsDecompressDoubleLossy(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize){ +static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); } #endif -static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, - char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, + char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsCompressTimestampImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { - int len = tsCompressTimestampImp(input, nelements, buffer); + int32_t len = tsCompressTimestampImp(input, nelements, buffer); return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); @@ -349,8 +372,9 @@ static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSi } } -static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output, - int outputSize, char algorithm, char *const buffer, int bufferSize) { +static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, + const int32_t nelements, char *const output, int32_t outputSize, + char algorithm, char *const buffer, int32_t bufferSize) { if (algorithm == ONE_STAGE_COMP) { return tsDecompressTimestampImp(input, nelements, output); } else if (algorithm == TWO_STAGE_COMP) { @@ -366,4 +390,4 @@ static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compr } #endif -#endif /*_TD_UTIL_COMPRESSION_H*/ \ No newline at end of file +#endif /*_TD_UTIL_COMPRESSION_H_*/ \ No newline at end of file diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 7dee01247b9e081585ab1ba4c4a1f907dec9cbc6..8275054a64f188c6f30a70a481f9ff9f9ab95d2b 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,85 +14,98 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_CONFIG_H -#define _TD_UTIL_CONFIG_H +#ifndef _TD_CONFIG_H_ +#define _TD_CONFIG_H_ + +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#define TSDB_CFG_MAX_NUM 115 -#define TSDB_CFG_PRINT_LEN 23 -#define TSDB_CFG_OPTION_LEN 24 -#define TSDB_CFG_VALUE_LEN 41 - -#define TSDB_CFG_CTYPE_B_CONFIG 1U // can be configured from file -#define TSDB_CFG_CTYPE_B_SHOW 2U // can displayed by "show configs" commands -#define TSDB_CFG_CTYPE_B_LOG 4U // is a log type configuration -#define TSDB_CFG_CTYPE_B_CLIENT 8U // can be displayed in the client log -#define TSDB_CFG_CTYPE_B_OPTION 16U // can be configured by taos_options function -#define TSDB_CFG_CTYPE_B_NOT_PRINT 32U // such as password - -#define MAX_FLOAT 100000 -#define MIN_FLOAT 0 - -enum { - TAOS_CFG_CSTATUS_NONE, // not configured - TAOS_CFG_CSTATUS_DEFAULT, // use system default value - TAOS_CFG_CSTATUS_FILE, // configured from file - TAOS_CFG_CSTATUS_OPTION, // configured by taos_options function - TAOS_CFG_CSTATUS_ARG, // configured by program argument -}; - -enum { - TAOS_CFG_VTYPE_INT8, - TAOS_CFG_VTYPE_INT16, - TAOS_CFG_VTYPE_INT32, - TAOS_CFG_VTYPE_UINT16, - TAOS_CFG_VTYPE_FLOAT, - TAOS_CFG_VTYPE_STRING, - TAOS_CFG_VTYPE_IPSTR, - TAOS_CFG_VTYPE_DIRECTORY, - TAOS_CFG_VTYPE_DATA_DIRCTORY, - TAOS_CFG_VTYPE_DOUBLE, -}; - -enum { - TAOS_CFG_UTYPE_NONE, - TAOS_CFG_UTYPE_PERCENT, - TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_MB, - TAOS_CFG_UTYPE_BYTE, - TAOS_CFG_UTYPE_SECOND, - TAOS_CFG_UTYPE_MS -}; +#define CFG_NAME_MAX_LEN 128 + +typedef enum { + CFG_STYPE_DEFAULT, + CFG_STYPE_CFG_FILE, + CFG_STYPE_ENV_FILE, + CFG_STYPE_ENV_VAR, + CFG_STYPE_APOLLO_URL, + CFG_STYPE_ARG_LIST, +} ECfgSrcType; + +typedef enum { + CFG_DTYPE_NONE, + CFG_DTYPE_BOOL, + CFG_DTYPE_INT32, + CFG_DTYPE_INT64, + CFG_DTYPE_FLOAT, + CFG_DTYPE_STRING, + CFG_DTYPE_DIR, + CFG_DTYPE_LOCALE, + CFG_DTYPE_CHARSET, + CFG_DTYPE_TIMEZONE +} ECfgDataType; + +typedef struct SConfigItem { + ECfgSrcType stype; + ECfgDataType dtype; + bool tsc; + char *name; + union { + bool bval; + float fval; + int32_t i32; + int64_t i64; + char *str; + }; + union { + int64_t imin; + double fmin; + }; + union { + int64_t imax; + double fmax; + }; + SArray *array; // SDiskCfg +} SConfigItem; typedef struct { - char * option; - void * ptr; - float minValue; - float maxValue; - int8_t cfgType; - int8_t cfgStatus; - int8_t unitType; - int8_t valType; - int32_t ptrLength; -} SGlobalCfg; - -extern SGlobalCfg tsGlobalConfig[]; -extern int32_t tsGlobalConfigNum; -extern char * tsCfgStatusStr[]; - -void taosReadGlobalLogCfg(); -int32_t taosReadCfgFromFile(); -void taosPrintCfg(); -void taosDumpGlobalCfg(); - -void taosAddConfigOption(SGlobalCfg cfg); -SGlobalCfg *taosGetConfigOption(const char *option); + const char *name; + const char *value; +} SConfigPair; + +typedef struct SConfig { + ECfgSrcType stype; + SArray *array; +} SConfig; + +SConfig *cfgInit(); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs); // SConfigPair +void cfgCleanup(SConfig *pCfg); + +int32_t cfgGetSize(SConfig *pCfg); +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); + +const char *cfgStypeStr(ECfgSrcType type); +const char *cfgDtypeStr(ECfgDataType type); + +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_CONFIG_H*/ +#endif /*_TD_CONFIG_H_*/ diff --git a/include/util/tcrc32c.h b/include/util/tcrc32c.h index 200103b173d44d73d0c640588e521ba9ae6f0ca0..0246e09c31140aa2fe943de868b8056f320be883 100644 --- a/include/util/tcrc32c.h +++ b/include/util/tcrc32c.h @@ -18,8 +18,10 @@ 3. This notice may not be removed or altered from any source distribution. */ -#ifndef _TD_UTIL_CRC32_H -#define _TD_UTIL_CRC32_H +#ifndef _TD_UTIL_CRC32_H_ +#define _TD_UTIL_CRC32_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -39,4 +41,4 @@ void taosResolveCRC(); } #endif -#endif /*_TD_UTIL_CRC32_H*/ +#endif /*_TD_UTIL_CRC32_H_*/ diff --git a/include/util/tdef.h b/include/util/tdef.h index d0f2b77f1fa92ee67ad2a2dce69c234be3b13cf7..a1b4fc11cf00eaf9615b41a50811b4574e759277 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -13,10 +13,8 @@ * along with this program. If not, see . */ -// clang-format off - -#ifndef _TD_UTIL_DEF_H -#define _TD_UTIL_DEF_H +#ifndef _TD_UTIL_DEF_H_ +#define _TD_UTIL_DEF_H_ #include "os.h" @@ -26,89 +24,106 @@ extern "C" { #define TSDB__packed -#define TSKEY int64_t -#define TSKEY_MIN INT64_MIN -#define TSKEY_MAX (INT64_MAX - 1) +#define TSKEY int64_t +#define TSKEY_MIN INT64_MIN +#define TSKEY_MAX (INT64_MAX - 1) #define TSKEY_INITIAL_VAL TSKEY_MIN // Bytes for each type. extern const int32_t TYPE_BYTES[15]; // TODO: replace and remove code below -#define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(int16_t) -#define INT_BYTES sizeof(int32_t) -#define LONG_BYTES sizeof(int64_t) -#define FLOAT_BYTES sizeof(float) -#define DOUBLE_BYTES sizeof(double) -#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) -#define TSDB_KEYSIZE sizeof(TSKEY) -#define TSDB_NCHAR_SIZE sizeof(int32_t) +#define CHAR_BYTES sizeof(char) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) // NULL definition -#define TSDB_DATA_BOOL_NULL 0x02 -#define TSDB_DATA_TINYINT_NULL 0x80 -#define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L -#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL - -#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN -#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF -#define TSDB_DATA_BINARY_NULL 0xFF - -#define TSDB_DATA_UTINYINT_NULL 0xFF -#define TSDB_DATA_USMALLINT_NULL 0xFFFF -#define TSDB_DATA_UINT_NULL 0xFFFFFFFF -#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL - -#define TSDB_DATA_NULL_STR "NULL" -#define TSDB_DATA_NULL_STR_L "null" - -#define TSDB_NETTEST_USER "nettestinternal" -#define TSDB_DEFAULT_USER "root" +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF + +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" + +#define TSDB_NETTEST_USER "nettestinternal" +#define TSDB_DEFAULT_USER "root" #ifdef _TD_POWER_ -#define TSDB_DEFAULT_PASS "powerdb" +#define TSDB_DEFAULT_PASS "powerdb" #elif (_TD_TQ_ == true) -#define TSDB_DEFAULT_PASS "tqueue" +#define TSDB_DEFAULT_PASS "tqueue" #elif (_TD_PRO_ == true) -#define TSDB_DEFAULT_PASS "prodb" +#define TSDB_DEFAULT_PASS "prodb" #else -#define TSDB_DEFAULT_PASS "taosdata" +#define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 +#define SHELL_MAX_PASSWORD_LEN 20 -#define TSDB_TRUE 1 -#define TSDB_FALSE 0 -#define TSDB_OK 0 +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 #define TSDB_ERR -1 #define TS_PATH_DELIMITER "." #define TS_ESCAPE_CHAR '`' -#define TSDB_TIME_PRECISION_MILLI 0 -#define TSDB_TIME_PRECISION_MICRO 1 -#define TSDB_TIME_PRECISION_NANO 2 +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 #define TSDB_TIME_PRECISION_MILLI_STR "ms" #define TSDB_TIME_PRECISION_MICRO_STR "us" #define TSDB_TIME_PRECISION_NANO_STR "ns" -#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) +#define TSDB_INFORMATION_SCHEMA_DB "information_schema" +#define TSDB_INS_TABLE_DNODES "dnodes" +#define TSDB_INS_TABLE_MNODES "mnodes" +#define TSDB_INS_TABLE_MODULES "modules" +#define TSDB_INS_TABLE_QNODES "qnodes" +#define TSDB_INS_TABLE_USER_DATABASE "user_database" +#define TSDB_INS_TABLE_USER_FUNCTIONS "user_functions" +#define TSDB_INS_TABLE_USER_INDEXES "user_indexes" +#define TSDB_INS_TABLE_USER_STABLES "user_stables" +#define TSDB_INS_TABLE_USER_STREAMS "user_streams" +#define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" +#define TSDB_INS_TABLE_USER_USERS "user_users" +#define TSDB_INS_TABLE_VGROUPS "vgroups" + +#define TSDB_TICK_PER_SECOND(precision) \ + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) -#define T_APPEND_MEMBER(dst, ptr, type, member) \ -do {\ - memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ - dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ -} while(0) -#define T_READ_MEMBER(src, type, target) \ -do { \ - (target) = *(type *)(src); \ - (src) = (void *)((char *)src + sizeof(type));\ -} while(0) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ + do { \ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member)); \ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member)); \ + } while (0) +#define T_READ_MEMBER(src, type, target) \ + do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type)); \ + } while (0) typedef enum EOperatorType { // arithmetic operator @@ -149,267 +164,269 @@ typedef enum EOperatorType { OP_TYPE_JSON_CONTAINS } EOperatorType; - typedef enum ELogicConditionType { LOGIC_COND_TYPE_AND, LOGIC_COND_TYPE_OR, LOGIC_COND_TYPE_NOT, } ELogicConditionType; +#define FUNCTION_CEIL 4500 +#define FUNCTION_FLOOR 4501 +#define FUNCTION_ABS 4502 +#define FUNCTION_ROUND 4503 -#define FUNCTION_CEIL 4500 -#define FUNCTION_FLOOR 4501 -#define FUNCTION_ABS 4502 -#define FUNCTION_ROUND 4503 - -#define FUNCTION_LENGTH 4800 -#define FUNCTION_CONCAT 4801 -#define FUNCTION_LTRIM 4802 -#define FUNCTION_RTRIM 4803 +#define FUNCTION_LENGTH 4800 +#define FUNCTION_CONCAT 4801 +#define FUNCTION_LTRIM 4802 +#define FUNCTION_RTRIM 4803 -#define TSDB_NAME_DELIMITER_LEN 1 +#define TSDB_NAME_DELIMITER_LEN 1 -#define TSDB_UNI_LEN 24 -#define TSDB_USER_LEN TSDB_UNI_LEN +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero -#define TSDB_ACCT_ID_LEN 11 - -#define TSDB_MAX_COLUMNS 4096 -#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns - -#define TSDB_NODE_NAME_LEN 64 -#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string -#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string -#define TSDB_DB_NAME_LEN 65 -#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) - -#define TSDB_FUNC_NAME_LEN 65 -#define TSDB_FUNC_COMMENT_LEN 4096 -#define TSDB_FUNC_CODE_LEN (65535 - 512) -#define TSDB_FUNC_BUF_SIZE 512 -#define TSDB_FUNC_TYPE_SCALAR 1 -#define TSDB_FUNC_TYPE_AGGREGATE 2 -#define TSDB_FUNC_MAX_RETRIEVE 1024 - -#define TSDB_TYPE_STR_MAX_LEN 32 -#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN -#define TSDB_CONSUMER_GROUP_LEN 192 -#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) -#define TSDB_COL_NAME_LEN 65 -#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 -#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE -#define TSDB_MAX_SQL_SHOW_LEN 1024 -#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb - -#define TSDB_APP_NAME_LEN TSDB_UNI_LEN -#define TSDB_STB_COMMENT_LEN 1024 - /** - * In some scenarios uint16_t (0~65535) is used to store the row len. - * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. - * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus - * the final value is 65531-(4096-1)*4 = 49151. - */ -#define TSDB_MAX_BYTES_PER_ROW 49151 -#define TSDB_MAX_TAGS_LEN 16384 -#define TSDB_MAX_TAGS 128 -#define TSDB_MAX_TAG_CONDITIONS 1024 - -#define TSDB_AUTH_LEN 16 -#define TSDB_PASSWORD_LEN 32 -#define TSDB_USET_PASSWORD_LEN 129 -#define TSDB_VERSION_LEN 12 -#define TSDB_LABEL_LEN 8 - -#define TSDB_CLUSTER_ID_LEN 40 -#define TSDB_FQDN_LEN 128 -#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) -#define TSDB_IPv4ADDR_LEN 16 -#define TSDB_FILENAME_LEN 128 -#define TSDB_SHOW_SQL_LEN 512 -#define TSDB_SHOW_SUBQUERY_LEN 1000 -#define TSDB_SLOW_QUERY_SQL_LEN 512 - -#define TSDB_TRANS_STAGE_LEN 12 -#define TSDB_TRANS_TYPE_LEN 16 -#define TSDB_TRANS_ERROR_LEN 64 - -#define TSDB_STEP_NAME_LEN 32 -#define TSDB_STEP_DESC_LEN 128 - -#define TSDB_ERROR_MSG_LEN 1024 -#define TSDB_DNODE_CONFIG_LEN 128 -#define TSDB_DNODE_VALUE_LEN 256 - -#define TSDB_MQTT_HOSTNAME_LEN 64 -#define TSDB_MQTT_PORT_LEN 8 -#define TSDB_MQTT_USER_LEN 24 -#define TSDB_MQTT_PASS_LEN 24 -#define TSDB_MQTT_TOPIC_LEN 64 -#define TSDB_MQTT_CLIENT_ID_LEN 32 - -#define TSDB_DB_TYPE_DEFAULT 0 -#define TSDB_DB_TYPE_TOPIC 1 - -#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE +#define TSDB_ACCT_ID_LEN 11 + +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns + +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 65 +#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) + +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_COMMENT_LEN 4096 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_FUNC_TYPE_SCALAR 1 +#define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 + +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_CONSUMER_GROUP_LEN 192 +#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_PARTITION_KEY_LEN (TSDB_CONSUMER_GROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 1024 +#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb + +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN +#define TSDB_STB_COMMENT_LEN 1024 +/** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 + +#define TSDB_AUTH_LEN 16 +#define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 +#define TSDB_VERSION_LEN 12 +#define TSDB_LABEL_LEN 8 + +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 + +#define TSDB_TRANS_STAGE_LEN 12 +#define TSDB_TRANS_TYPE_LEN 16 +#define TSDB_TRANS_ERROR_LEN 64 + +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + +#define TSDB_ERROR_MSG_LEN 1024 +#define TSDB_DNODE_CONFIG_LEN 128 +#define TSDB_DNODE_VALUE_LEN 256 + +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 + +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + +#define TSDB_DEFAULT_PKT_SIZE 65480 // same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE -#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value -#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 16 #define TSDB_MAX_VNODES 512 #define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MAX_VNODES_PER_DB 4096 -#define TSDB_DEFAULT_VN_PER_DB 2 - -#define TSDB_DNODE_ROLE_ANY 0 -#define TSDB_DNODE_ROLE_MGMT 1 -#define TSDB_DNODE_ROLE_VNODE 2 +#define TSDB_DEFAULT_VN_PER_DB 2 -#define TSDB_MAX_REPLICA 5 +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 -#define TSDB_TBNAME_COLUMN_INDEX (-1) -#define TSDB_UD_COLUMN_INDEX (-1000) -#define TSDB_RES_COL_ID (-5000) +#define TSDB_MAX_REPLICA 5 -#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - -#define TSDB_MIN_CACHE_BLOCK_SIZE 1 -#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode -#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) -#define TSDB_MIN_TOTAL_BLOCKS 3 -#define TSDB_MAX_TOTAL_BLOCKS 10000 -#define TSDB_DEFAULT_TOTAL_BLOCKS 6 +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta -#define TSDB_MIN_DAYS_PER_FILE 1 -#define TSDB_MAX_DAYS_PER_FILE 3650 -#define TSDB_DEFAULT_DAYS_PER_FILE 10 +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 -#define TSDB_MIN_KEEP 1 // data in db to be reserved. -#define TSDB_MAX_KEEP 365000 // data in db to be reserved. -#define TSDB_DEFAULT_KEEP 3650 // ten years +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 -#define TSDB_MIN_MIN_ROW_FBLOCK 10 -#define TSDB_MAX_MIN_ROW_FBLOCK 1000 -#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 -#define TSDB_MIN_MAX_ROW_FBLOCK 200 -#define TSDB_MAX_MAX_ROW_FBLOCK 10000 -#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years -#define TSDB_MIN_COMMIT_TIME 30 -#define TSDB_MAX_COMMIT_TIME 40960 -#define TSDB_DEFAULT_COMMIT_TIME 3600 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 -#define TSDB_MIN_FSYNC_PERIOD 0 -#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond -#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 -#define TSDB_MIN_WAL_LEVEL 0 -#define TSDB_MAX_WAL_LEVEL 2 -#define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 -#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO -#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second -#define TSDB_MIN_COMP_LEVEL 0 -#define TSDB_MAX_COMP_LEVEL 2 -#define TSDB_DEFAULT_COMP_LEVEL 2 +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 -#define TSDB_MIN_DB_REPLICA_OPTION 1 -#define TSDB_MAX_DB_REPLICA_OPTION 3 -#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI -#define TSDB_MIN_DB_QUORUM_OPTION 1 -#define TSDB_MAX_DB_QUORUM_OPTION 2 -#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 -#define TSDB_MIN_DB_UPDATE 0 -#define TSDB_MAX_DB_UPDATE 2 -#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 -#define TSDB_MIN_DB_CACHE_LAST_ROW 0 -#define TSDB_MAX_DB_CACHE_LAST_ROW 3 -#define TSDB_DEFAULT_CACHE_LAST_ROW 0 +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 -#define TSDB_MAX_JOIN_TABLE_NUM 10 -#define TSDB_MAX_UNION_CLAUSE 5 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 -#define TSDB_MAX_FIELD_LEN 16384 -#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 -#define PRIMARYKEY_TIMESTAMP_COL_ID 1 -#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 -#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 -#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type -#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN - TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_ID 1 +#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) -#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default +#define TSDB_MAX_RPC_THREADS 5 +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default /* * 1. ordinary sub query for select * from super_table * 2. all sqlobj generated by createSubqueryObj with this flag */ -#define TSDB_QUERY_TYPE_SUBQUERY 0x02u -#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table - -#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side -#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table -#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query -#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query -#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage - -#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u -#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type -#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u -#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file -#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type -#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query - -#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) -#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) -#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) -#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) - -#define TSDB_ORDER_ASC 1 -#define TSDB_ORDER_DESC 2 - -#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 -#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 -#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 -#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 -#define TSDB_DEFAULT_USERS_HASH_SIZE 20 -#define TSDB_DEFAULT_DBS_HASH_SIZE 100 -#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 -#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 -#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 - -#define TSDB_MAX_WAL_SIZE (1024*1024*3) - -#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P - -#define TFS_MAX_TIERS 3 +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage + +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query + +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_MAX_WAL_SIZE (1024 * 1024 * 3) + +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +#define TFS_MAX_TIERS 3 #define TFS_MAX_DISKS_PER_TIER 16 -#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) -#define TFS_MIN_LEVEL 0 -#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) -#define TFS_PRIMARY_LEVEL 0 -#define TFS_PRIMARY_ID 0 +#define TFS_MAX_DISKS (TFS_MAX_TIERS * TFS_MAX_DISKS_PER_TIER) +#define TFS_MIN_LEVEL 0 +#define TFS_MAX_LEVEL (TFS_MAX_TIERS - 1) +#define TFS_PRIMARY_LEVEL 0 +#define TFS_PRIMARY_ID 0 #define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024 enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED }; enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK }; +typedef struct { + char dir[TSDB_FILENAME_LEN]; + int32_t level; + int32_t primary; +} SDiskCfg; + #ifdef __cplusplus } #endif diff --git a/include/common/tcfg.h b/include/util/tdes.h similarity index 77% rename from include/common/tcfg.h rename to include/util/tdes.h index 7ccfd4b0f5d9a739016f506e7a121276a1cac2a5..edb325da28ebdaa4085c1580fb320ea0bdd0a2b0 100644 --- a/include/common/tcfg.h +++ b/include/util/tdes.h @@ -13,23 +13,20 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_CFG_H_ -#define _TD_COMMON_CFG_H_ +#ifndef _TD_UTIL_DES_H +#define _TD_UTIL_DES_H + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "tdef.h" - -typedef struct { - char dir[TSDB_FILENAME_LEN]; - int level; - int primary; -} SDiskCfg; +char *taosDesEncode(int64_t key, char *src, int32_t len); +char *taosDesDecode(int64_t key, char *src, int32_t len); #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_CFG_H_*/ +#endif /*_TD_UTIL_DES_H*/ \ No newline at end of file diff --git a/include/util/encode.h b/include/util/tencode.h similarity index 58% rename from include/util/encode.h rename to include/util/tencode.h index ba63759737f250e120390dc1c93e676cbb345824..c058eebb5016141dc102aeed01621b7f4cb368fc 100644 --- a/include/util/encode.h +++ b/include/util/tencode.h @@ -16,8 +16,8 @@ #ifndef _TD_UTIL_ENCODE_H_ #define _TD_UTIL_ENCODE_H_ -#include "freelist.h" #include "tcoding.h" +#include "tfreelist.h" #include "tmacro.h" #ifdef __cplusplus @@ -71,62 +71,62 @@ typedef struct { TD_SLIST(SCoderNode) stack; } SCoder; -#define TD_CODER_POS(CODER) ((CODER)->pos) -#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) -#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) +#define TD_CODER_POS(CODER) ((CODER)->pos) +#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) +#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) -#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) +#define TCODER_MALLOC(SIZE, CODER) TFL_MALLOC(SIZE, &((CODER)->fl)) void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderClear(SCoder* pCoder); /* ------------------------ ENCODE ------------------------ */ -int tStartEncode(SCoder* pEncoder); -void tEndEncode(SCoder* pEncoder); -static int tEncodeU8(SCoder* pEncoder, uint8_t val); -static int tEncodeI8(SCoder* pEncoder, int8_t val); -static int tEncodeU16(SCoder* pEncoder, uint16_t val); -static int tEncodeI16(SCoder* pEncoder, int16_t val); -static int tEncodeU32(SCoder* pEncoder, uint32_t val); -static int tEncodeI32(SCoder* pEncoder, int32_t val); -static int tEncodeU64(SCoder* pEncoder, uint64_t val); -static int tEncodeI64(SCoder* pEncoder, int64_t val); -static int tEncodeU16v(SCoder* pEncoder, uint16_t val); -static int tEncodeI16v(SCoder* pEncoder, int16_t val); -static int tEncodeU32v(SCoder* pEncoder, uint32_t val); -static int tEncodeI32v(SCoder* pEncoder, int32_t val); -static int tEncodeU64v(SCoder* pEncoder, uint64_t val); -static int tEncodeI64v(SCoder* pEncoder, int64_t val); -static int tEncodeFloat(SCoder* pEncoder, float val); -static int tEncodeDouble(SCoder* pEncoder, double val); -static int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); -static int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); -static int tEncodeCStr(SCoder* pEncoder, const char* val); +int32_t tStartEncode(SCoder* pEncoder); +void tEndEncode(SCoder* pEncoder); +static int32_t tEncodeU8(SCoder* pEncoder, uint8_t val); +static int32_t tEncodeI8(SCoder* pEncoder, int8_t val); +static int32_t tEncodeU16(SCoder* pEncoder, uint16_t val); +static int32_t tEncodeI16(SCoder* pEncoder, int16_t val); +static int32_t tEncodeU32(SCoder* pEncoder, uint32_t val); +static int32_t tEncodeI32(SCoder* pEncoder, int32_t val); +static int32_t tEncodeU64(SCoder* pEncoder, uint64_t val); +static int32_t tEncodeI64(SCoder* pEncoder, int64_t val); +static int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val); +static int32_t tEncodeI16v(SCoder* pEncoder, int16_t val); +static int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val); +static int32_t tEncodeI32v(SCoder* pEncoder, int32_t val); +static int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val); +static int32_t tEncodeI64v(SCoder* pEncoder, int64_t val); +static int32_t tEncodeFloat(SCoder* pEncoder, float val); +static int32_t tEncodeDouble(SCoder* pEncoder, double val); +static int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len); +static int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len); +static int32_t tEncodeCStr(SCoder* pEncoder, const char* val); /* ------------------------ DECODE ------------------------ */ -int tStartDecode(SCoder* pDecoder); -void tEndDecode(SCoder* pDecoder); -static bool tDecodeIsEnd(SCoder* pCoder); -static int tDecodeU8(SCoder* pDecoder, uint8_t* val); -static int tDecodeI8(SCoder* pDecoder, int8_t* val); -static int tDecodeU16(SCoder* pDecoder, uint16_t* val); -static int tDecodeI16(SCoder* pDecoder, int16_t* val); -static int tDecodeU32(SCoder* pDecoder, uint32_t* val); -static int tDecodeI32(SCoder* pDecoder, int32_t* val); -static int tDecodeU64(SCoder* pDecoder, uint64_t* val); -static int tDecodeI64(SCoder* pDecoder, int64_t* val); -static int tDecodeU16v(SCoder* pDecoder, uint16_t* val); -static int tDecodeI16v(SCoder* pDecoder, int16_t* val); -static int tDecodeU32v(SCoder* pDecoder, uint32_t* val); -static int tDecodeI32v(SCoder* pDecoder, int32_t* val); -static int tDecodeU64v(SCoder* pDecoder, uint64_t* val); -static int tDecodeI64v(SCoder* pDecoder, int64_t* val); -static int tDecodeFloat(SCoder* pDecoder, float* val); -static int tDecodeDouble(SCoder* pDecoder, double* val); -static int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); -static int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); -static int tDecodeCStr(SCoder* pDecoder, const char** val); -static int tDecodeCStrTo(SCoder* pDecoder, char* val); +int32_t tStartDecode(SCoder* pDecoder); +void tEndDecode(SCoder* pDecoder); +static bool tDecodeIsEnd(SCoder* pCoder); +static int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val); +static int32_t tDecodeI8(SCoder* pDecoder, int8_t* val); +static int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val); +static int32_t tDecodeI16(SCoder* pDecoder, int16_t* val); +static int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val); +static int32_t tDecodeI32(SCoder* pDecoder, int32_t* val); +static int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val); +static int32_t tDecodeI64(SCoder* pDecoder, int64_t* val); +static int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val); +static int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val); +static int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val); +static int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val); +static int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val); +static int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val); +static int32_t tDecodeFloat(SCoder* pDecoder, float* val); +static int32_t tDecodeDouble(SCoder* pDecoder, double* val); +static int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len); +static int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len); +static int32_t tDecodeCStr(SCoder* pDecoder, const char** val); +static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val); /* ------------------------ IMPL ------------------------ */ #define TD_ENCODE_MACRO(CODER, VAL, TYPE, BITS) \ @@ -190,7 +190,7 @@ static int tDecodeCStrTo(SCoder* pDecoder, char* val); return 0; // 8 -static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) { +static FORCE_INLINE int32_t tEncodeU8(SCoder* pEncoder, uint8_t val) { if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(uint8_t, TD_CODER_CURRENT(pEncoder), val); @@ -199,7 +199,7 @@ static FORCE_INLINE int tEncodeU8(SCoder* pEncoder, uint8_t val) { return 0; } -static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) { +static FORCE_INLINE int32_t tEncodeI8(SCoder* pEncoder, int8_t val) { if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, sizeof(val))) return -1; tPut(int8_t, TD_CODER_CURRENT(pEncoder), val); @@ -209,31 +209,31 @@ static FORCE_INLINE int tEncodeI8(SCoder* pEncoder, int8_t val) { } // 16 -static FORCE_INLINE int tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); } -static FORCE_INLINE int tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tEncodeU16(SCoder* pEncoder, uint16_t val) { TD_ENCODE_MACRO(pEncoder, val, uint16_t, 16); } +static FORCE_INLINE int32_t tEncodeI16(SCoder* pEncoder, int16_t val) { TD_ENCODE_MACRO(pEncoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); } -static FORCE_INLINE int tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tEncodeU32(SCoder* pEncoder, uint32_t val) { TD_ENCODE_MACRO(pEncoder, val, uint32_t, 32); } +static FORCE_INLINE int32_t tEncodeI32(SCoder* pEncoder, int32_t val) { TD_ENCODE_MACRO(pEncoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); } -static FORCE_INLINE int tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tEncodeU64(SCoder* pEncoder, uint64_t val) { TD_ENCODE_MACRO(pEncoder, val, uint64_t, 64); } +static FORCE_INLINE int32_t tEncodeI64(SCoder* pEncoder, int64_t val) { TD_ENCODE_MACRO(pEncoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI16v(SCoder* pEncoder, int16_t val) { +static FORCE_INLINE int32_t tEncodeU16v(SCoder* pEncoder, uint16_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI16v(SCoder* pEncoder, int16_t val) { return tEncodeU16v(pEncoder, ZIGZAGE(int16_t, val)); } // 32v -static FORCE_INLINE int tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI32v(SCoder* pEncoder, int32_t val) { +static FORCE_INLINE int32_t tEncodeU32v(SCoder* pEncoder, uint32_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI32v(SCoder* pEncoder, int32_t val) { return tEncodeU32v(pEncoder, ZIGZAGE(int32_t, val)); } // 64v -static FORCE_INLINE int tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } -static FORCE_INLINE int tEncodeI64v(SCoder* pEncoder, int64_t val) { +static FORCE_INLINE int32_t tEncodeU64v(SCoder* pEncoder, uint64_t val) { TD_ENCODE_VARIANT_MACRO(pEncoder, val); } +static FORCE_INLINE int32_t tEncodeI64v(SCoder* pEncoder, int64_t val) { return tEncodeU64v(pEncoder, ZIGZAGE(int64_t, val)); } -static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { +static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) { union { uint32_t ui; float f; @@ -242,7 +242,7 @@ static FORCE_INLINE int tEncodeFloat(SCoder* pEncoder, float val) { return tEncodeU32(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { +static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) { union { uint64_t ui; double d; @@ -251,7 +251,7 @@ static FORCE_INLINE int tEncodeDouble(SCoder* pEncoder, double val) { return tEncodeU64(pEncoder, v.ui); } -static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) { +static FORCE_INLINE int32_t tEncodeBinary(SCoder* pEncoder, const void* val, uint64_t len) { if (tEncodeU64v(pEncoder, len) < 0) return -1; if (pEncoder->data) { if (TD_CODER_CHECK_CAPACITY_FAILED(pEncoder, len)) return -1; @@ -262,24 +262,24 @@ static FORCE_INLINE int tEncodeBinary(SCoder* pEncoder, const void* val, uint64_ return 0; } -static FORCE_INLINE int tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { +static FORCE_INLINE int32_t tEncodeCStrWithLen(SCoder* pEncoder, const char* val, uint64_t len) { return tEncodeBinary(pEncoder, (void*)val, len + 1); } -static FORCE_INLINE int tEncodeCStr(SCoder* pEncoder, const char* val) { +static FORCE_INLINE int32_t tEncodeCStr(SCoder* pEncoder, const char* val) { return tEncodeCStrWithLen(pEncoder, val, (uint64_t)strlen(val)); } /* ------------------------ FOR DECODER ------------------------ */ // 8 -static FORCE_INLINE int tDecodeU8(SCoder* pDecoder, uint8_t* val) { +static FORCE_INLINE int32_t tDecodeU8(SCoder* pDecoder, uint8_t* val) { if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(uint8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); return 0; } -static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) { +static FORCE_INLINE int32_t tDecodeI8(SCoder* pDecoder, int8_t* val) { if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, sizeof(*val))) return -1; tGet(int8_t, TD_CODER_CURRENT(pDecoder), *val); TD_CODER_MOVE_POS(pDecoder, sizeof(*val)); @@ -287,21 +287,27 @@ static FORCE_INLINE int tDecodeI8(SCoder* pDecoder, int8_t* val) { } // 16 -static FORCE_INLINE int tDecodeU16(SCoder* pDecoder, uint16_t* val) { TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); } -static FORCE_INLINE int tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); } +static FORCE_INLINE int32_t tDecodeU16(SCoder* pDecoder, uint16_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint16_t, 16); +} +static FORCE_INLINE int32_t tDecodeI16(SCoder* pDecoder, int16_t* val) { TD_DECODE_MACRO(pDecoder, val, int16_t, 16); } // 32 -static FORCE_INLINE int tDecodeU32(SCoder* pDecoder, uint32_t* val) { TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); } -static FORCE_INLINE int tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); } +static FORCE_INLINE int32_t tDecodeU32(SCoder* pDecoder, uint32_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint32_t, 32); +} +static FORCE_INLINE int32_t tDecodeI32(SCoder* pDecoder, int32_t* val) { TD_DECODE_MACRO(pDecoder, val, int32_t, 32); } // 64 -static FORCE_INLINE int tDecodeU64(SCoder* pDecoder, uint64_t* val) { TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); } -static FORCE_INLINE int tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); } +static FORCE_INLINE int32_t tDecodeU64(SCoder* pDecoder, uint64_t* val) { + TD_DECODE_MACRO(pDecoder, val, uint64_t, 64); +} +static FORCE_INLINE int32_t tDecodeI64(SCoder* pDecoder, int64_t* val) { TD_DECODE_MACRO(pDecoder, val, int64_t, 64); } // 16v -static FORCE_INLINE int tDecodeU16v(SCoder* pDecoder, uint16_t* val) { +static FORCE_INLINE int32_t tDecodeU16v(SCoder* pDecoder, uint16_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint16_t); } -static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) { +static FORCE_INLINE int32_t tDecodeI16v(SCoder* pDecoder, int16_t* val) { uint16_t tval; if (tDecodeU16v(pDecoder, &tval) < 0) { return -1; @@ -311,11 +317,11 @@ static FORCE_INLINE int tDecodeI16v(SCoder* pDecoder, int16_t* val) { } // 32v -static FORCE_INLINE int tDecodeU32v(SCoder* pDecoder, uint32_t* val) { +static FORCE_INLINE int32_t tDecodeU32v(SCoder* pDecoder, uint32_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint32_t); } -static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) { +static FORCE_INLINE int32_t tDecodeI32v(SCoder* pDecoder, int32_t* val) { uint32_t tval; if (tDecodeU32v(pDecoder, &tval) < 0) { return -1; @@ -325,11 +331,11 @@ static FORCE_INLINE int tDecodeI32v(SCoder* pDecoder, int32_t* val) { } // 64v -static FORCE_INLINE int tDecodeU64v(SCoder* pDecoder, uint64_t* val) { +static FORCE_INLINE int32_t tDecodeU64v(SCoder* pDecoder, uint64_t* val) { TD_DECODE_VARIANT_MACRO(pDecoder, val, uint64_t); } -static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { +static FORCE_INLINE int32_t tDecodeI64v(SCoder* pDecoder, int64_t* val) { uint64_t tval; if (tDecodeU64v(pDecoder, &tval) < 0) { return -1; @@ -338,7 +344,7 @@ static FORCE_INLINE int tDecodeI64v(SCoder* pDecoder, int64_t* val) { return 0; } -static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { +static FORCE_INLINE int32_t tDecodeFloat(SCoder* pDecoder, float* val) { union { uint32_t ui; float f; @@ -352,7 +358,7 @@ static FORCE_INLINE int tDecodeFloat(SCoder* pDecoder, float* val) { return 0; } -static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { +static FORCE_INLINE int32_t tDecodeDouble(SCoder* pDecoder, double* val) { union { uint64_t ui; double d; @@ -366,7 +372,7 @@ static FORCE_INLINE int tDecodeDouble(SCoder* pDecoder, double* val) { return 0; } -static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { +static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, uint64_t* len) { if (tDecodeU64v(pDecoder, len) < 0) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; @@ -376,18 +382,18 @@ static FORCE_INLINE int tDecodeBinary(SCoder* pDecoder, const void** val, uint64 return 0; } -static FORCE_INLINE int tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { +static FORCE_INLINE int32_t tDecodeCStrAndLen(SCoder* pDecoder, const char** val, uint64_t* len) { if (tDecodeBinary(pDecoder, (const void**)val, len) < 0) return -1; (*len) -= 1; return 0; } -static FORCE_INLINE int tDecodeCStr(SCoder* pDecoder, const char** val) { +static FORCE_INLINE int32_t tDecodeCStr(SCoder* pDecoder, const char** val) { uint64_t len; return tDecodeCStrAndLen(pDecoder, val, &len); } -static int tDecodeCStrTo(SCoder* pDecoder, char* val) { +static int32_t tDecodeCStrTo(SCoder* pDecoder, char* val) { const char* pStr; uint64_t len; if (tDecodeCStrAndLen(pDecoder, &pStr, &len) < 0) return -1; diff --git a/include/util/texception.h b/include/util/texception.h new file mode 100644 index 0000000000000000000000000000000000000000..576545d96c3641499082c0d86d8fb689829726d9 --- /dev/null +++ b/include/util/texception.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2020 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_EXCEPTION_H_ +#define _TD_UTIL_EXCEPTION_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * cleanup actions + */ +typedef struct SCleanupAction { + bool failOnly; + uint8_t wrapper; + uint16_t reserved; + void* func; + union { + void* Ptr; + bool Bool; + char Char; + int8_t Int8; + uint8_t Uint8; + int16_t Int16; + uint16_t Uint16; + int32_t Int; + uint32_t Uint; + int32_t Int32; + uint32_t Uint32; + int64_t Int64; + uint64_t Uint64; + float Float; + double Double; + } arg1, arg2; +} SCleanupAction; + +/* + * exception hander registration + */ +typedef struct SExceptionNode { + struct SExceptionNode* prev; + jmp_buf jb; + int32_t code; + int32_t maxCleanupAction; + int32_t numCleanupAction; + SCleanupAction* cleanupActions; +} SExceptionNode; + +// functions & macros for auto-cleanup + +void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2); +void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2); +void cleanupPush_void_ptr(bool failOnly, void* func, void* arg); +void cleanupPush_int_int(bool failOnly, void* func, int32_t arg); +void cleanupPush_void(bool failOnly, void* func); +void cleanupPush_int_ptr(bool failOnly, void* func, void* arg); + +int32_t cleanupGetActionCount(); +void cleanupExecuteTo(int32_t anchor, bool failed); +void cleanupExecute(SExceptionNode* node, bool failed); +bool cleanupExceedLimit(); + +#define CLEANUP_PUSH_VOID_PTR_PTR(failOnly, func, arg1, arg2) \ + cleanupPush_void_ptr_ptr((failOnly), (void*)(func), (void*)(arg1), (void*)(arg2)) +#define CLEANUP_PUSH_VOID_PTR_BOOL(failOnly, func, arg1, arg2) \ + cleanupPush_void_ptr_bool((failOnly), (void*)(func), (void*)(arg1), (bool)(arg2)) +#define CLEANUP_PUSH_VOID_PTR(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (void*)(arg)) +#define CLEANUP_PUSH_INT_INT(failOnly, func, arg) cleanupPush_void_ptr((failOnly), (void*)(func), (int32_t)(arg)) +#define CLEANUP_PUSH_VOID(failOnly, func) cleanupPush_void((failOnly), (void*)(func)) +#define CLEANUP_PUSH_INT_PTR(failOnly, func, arg) cleanupPush_int_ptr((failOnly), (void*)(func), (void*)(arg)) +#define CLEANUP_PUSH_FREE(failOnly, arg) cleanupPush_void_ptr((failOnly), free, (void*)(arg)) +#define CLEANUP_PUSH_CLOSE(failOnly, arg) cleanupPush_int_int((failOnly), close, (int32_t)(arg)) +#define CLEANUP_PUSH_FCLOSE(failOnly, arg) cleanupPush_int_ptr((failOnly), fclose, (void*)(arg)) + +#define CLEANUP_GET_ANCHOR() cleanupGetActionCount() +#define CLEANUP_EXECUTE_TO(anchor, failed) cleanupExecuteTo((anchor), (failed)) +#define CLEANUP_EXCEED_LIMIT() cleanupExceedLimit() + +// functions & macros for exception handling + +void exceptionPushNode(SExceptionNode* node); +int32_t exceptionPopNode(); +void exceptionThrow(int32_t code); + +#define TRY(maxCleanupActions) \ + do { \ + SExceptionNode exceptionNode = {0}; \ + SCleanupAction cleanupActions[(maxCleanupActions) > 0 ? (maxCleanupActions) : 1]; \ + exceptionNode.maxCleanupAction = (maxCleanupActions) > 0 ? (maxCleanupActions) : 1; \ + exceptionNode.cleanupActions = cleanupActions; \ + exceptionPushNode(&exceptionNode); \ + int32_t caughtException = setjmp(exceptionNode.jb); \ + if (caughtException == 0) + +#define CATCH(code) \ + int32_t code = exceptionPopNode(); \ + if (caughtException == 1) + +#define FINALLY(code) int32_t code = exceptionPopNode(); + +#define END_TRY \ + } \ + while (0) \ + ; + +#define THROW(x) exceptionThrow((x)) +#define CAUGHT_EXCEPTION() ((bool)(caughtException == 1)) +#define CLEANUP_EXECUTE() cleanupExecute(&exceptionNode, CAUGHT_EXCEPTION()) + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_EXCEPTION_H_*/ diff --git a/include/util/tfile.h b/include/util/tfile.h deleted file mode 100644 index d3813051a4fb62777c9df0c442617b267896e16c..0000000000000000000000000000000000000000 --- a/include/util/tfile.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_FILE_H -#define _TD_UTIL_FILE_H - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// init taos file module -int32_t tfInit(); - -// clean up taos file module -void tfCleanup(); - -// the same syntax as UNIX standard open/close/read/write -// but FD is int64_t and will never be reused -int64_t tfOpenRead(const char *pathname); -int64_t tfOpenReadWrite(const char *pathname); -int64_t tfOpenCreateWrite(const char *pathname); -int64_t tfOpenCreateWriteAppend(const char *pathname); - -int64_t tfClose(int64_t tfd); -int64_t tfWrite(int64_t tfd, void *buf, int64_t count); -int64_t tfRead(int64_t tfd, void *buf, int64_t count); -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int64_t offset); -int32_t tfFsync(int64_t tfd); -bool tfValid(int64_t tfd); -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); -int32_t tfFtruncate(int64_t tfd, int64_t length); -void * tfMmapReadOnly(int64_t tfd, int64_t length); -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_FILE_H*/ diff --git a/include/util/freelist.h b/include/util/tfreelist.h similarity index 97% rename from include/util/freelist.h rename to include/util/tfreelist.h index 497a6d58c3f0791a90765f0682f5474ca8a2b6cf..c1913ebaf95866218db3d6b151a903f040e2175e 100644 --- a/include/util/freelist.h +++ b/include/util/tfreelist.h @@ -16,7 +16,6 @@ #ifndef _TD_UTIL_FREELIST_H_ #define _TD_UTIL_FREELIST_H_ -#include "os.h" #include "tlist.h" #ifdef __cplusplus diff --git a/include/util/tfunctional.h b/include/util/tfunctional.h index c96f997f06c363bfb06b8177a50b801a0cfbaea7..43e3cd5e483a37bad242a75799a836e5433012ce 100644 --- a/include/util/tfunctional.h +++ b/include/util/tfunctional.h @@ -12,45 +12,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_FUNCTIONAL_H -#define _TD_UTIL_FUNCTIONAL_H +#ifndef _TD_UTIL_FUNCTIONAL_H_ +#define _TD_UTIL_FUNCTIONAL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - -//TODO: hard to use, trying to rewrite it using va_list +// TODO: hard to use, trying to rewrite it using va_list typedef void* (*GenericVaFunc)(void* args[]); -typedef int32_t (*I32VaFunc) (void* args[]); -typedef void (*VoidVaFunc) (void* args[]); +typedef int32_t (*I32VaFunc)(void* args[]); +typedef void (*VoidVaFunc)(void* args[]); typedef struct GenericSavedFunc { GenericVaFunc func; - void * args[]; + void* args[]; } tGenericSavedFunc; typedef struct I32SavedFunc { I32VaFunc func; - void * args[]; + void* args[]; } tI32SavedFunc; typedef struct VoidSavedFunc { VoidVaFunc func; - void * args[]; + void* args[]; } tVoidSavedFunc; -tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs); -tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs); -tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs); -void* genericInvoke(tGenericSavedFunc* const pSavedFunc); -int32_t i32Invoke(tI32SavedFunc* const pSavedFunc); -void voidInvoke(tVoidSavedFunc* const pSavedFunc); +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs); +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs); +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs); +void* genericInvoke(tGenericSavedFunc* const pSavedFunc); +int32_t i32Invoke(tI32SavedFunc* const pSavedFunc); +void voidInvoke(tVoidSavedFunc* const pSavedFunc); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_FUNCTIONAL_H*/ +#endif /*_TD_UTIL_FUNCTIONAL_H_*/ diff --git a/include/util/thash.h b/include/util/thash.h index 1f72045a830e150e555f66fe36e0d453e680eff6..5c344f3f0fc5e59cafca0c461639cab9ae052225 100644 --- a/include/util/thash.h +++ b/include/util/thash.h @@ -13,22 +13,22 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_HASH_H -#define TDENGINE_HASH_H +#ifndef _TD_UTIL_HASH_H_ +#define _TD_UTIL_HASH_H_ + +#include "tarray.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif -#include "tarray.h" -#include "tlockfree.h" - typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); -typedef int32_t (*_equal_fn_t)(const void*, const void*, size_t len); +typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len); typedef void (*_hash_before_fn_t)(void *); typedef void (*_hash_free_fn_t)(void *); -#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_DEFAULT_LOAD_FACTOR (0.75) #define HASH_INDEX(v, c) ((v) & ((c)-1)) @@ -59,43 +59,43 @@ _equal_fn_t taosGetDefaultEqualFunction(int32_t type); typedef struct SHashNode { struct SHashNode *next; - uint32_t hashVal; // the hash value of key - uint32_t dataLen; // length of data - uint32_t keyLen; // length of the key - uint16_t count; // reference count - int8_t removed; // flag to indicate removed + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t count; // reference count + int8_t removed; // flag to indicate removed char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) +#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) typedef enum SHashLockTypeE { - HASH_NO_LOCK = 0, - HASH_ENTRY_LOCK = 1, + HASH_NO_LOCK = 0, + HASH_ENTRY_LOCK = 1, } SHashLockTypeE; typedef struct SHashEntry { - int32_t num; // number of elements in current entry - SRWLatch latch; // entry latch + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch SHashNode *next; } SHashEntry; typedef struct SHashObj { - SHashEntry **hashList; - uint32_t capacity; // number of slots - uint32_t size; // number of elements in hash table - - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function - _equal_fn_t equalFp; // equal function - _hash_before_fn_t callbackFp; // function invoked before return the value to caller - - SRWLatch lock; // read-write spin lock - SHashLockTypeE type; // lock type - bool enableUpdate; // enable update - SArray *pMemBlock; // memory block allocated for SHashEntry + SHashEntry **hashList; + uint32_t capacity; // number of slots + uint32_t size; // number of elements in hash table + + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + _equal_fn_t equalFp; // equal function + _hash_before_fn_t callbackFp; // function invoked before return the value to caller + + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; /** @@ -128,7 +128,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded); - /** * return the payload data with the specified key * @@ -147,7 +146,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param destBuf * @return */ -void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* destBuf); +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf); /** * Clone the result to interval allocated buffer @@ -157,7 +156,7 @@ void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* * @param destBuf * @return */ -void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz); +void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz); /** * remove item with the specified key @@ -206,15 +205,14 @@ void *taosHashIterate(SHashObj *pHashObj, void *p); * @param pHashObj * @param p */ -void taosHashCancelIterate(SHashObj *pHashObj, void *p); +void taosHashCancelIterate(SHashObj *pHashObj, void *p); /** * Get the corresponding key information for a given data in hash table * @param data * @return */ -int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); - +int32_t taosHashGetKey(void *data, void **key, size_t *keyLen); /** * Get the corresponding key information for a given data in hash table, using memcpy @@ -222,13 +220,13 @@ int32_t taosHashGetKey(void *data, void** key, size_t* keyLen); * @param dst * @return */ -static FORCE_INLINE int32_t taosHashCopyKey(void *data, void* dst) { +static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) { if (NULL == data || NULL == dst) { return -1; } - - SHashNode * node = GET_HASH_PNODE(data); - void* key = GET_HASH_NODE_KEY(node); + + SHashNode *node = GET_HASH_PNODE(data); + void *key = GET_HASH_NODE_KEY(node); memcpy(dst, key, node->keyLen); return 0; @@ -249,7 +247,7 @@ int32_t taosHashGetDataLen(void *data); * @param keyLen * @return */ -void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen); /** * release the prevous acquired obj @@ -262,9 +260,8 @@ void taosHashRelease(SHashObj *pHashObj, void *p); void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp); - #ifdef __cplusplus } #endif -#endif // TDENGINE_HASH_H +#endif // _TD_UTIL_HASH_H_ diff --git a/include/util/theap.h b/include/util/theap.h index fd1a39f8dd654338edf219531eaf982c0898df79..fb5ff8301a9b08b2cb53c353a363042c2de9cc34 100644 --- a/include/util/theap.h +++ b/include/util/theap.h @@ -12,19 +12,20 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef TDENGINE_HEAP_H -#define TDENGINE_HEAP_H + +#ifndef _TD_UTIL_HEAP_H_ +#define _TD_UTIL_HEAP_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" - struct HeapNode; /* Return non-zero if a < b. */ -typedef int (*HeapCompareFn)(const struct HeapNode* a, const struct HeapNode* b); +typedef int32_t (*HeapCompareFn)(const struct HeapNode* a, const struct HeapNode* b); typedef struct HeapNode { struct HeapNode* left; @@ -38,15 +39,14 @@ typedef struct HeapNode { * */ typedef struct { - HeapNode* min; - size_t nelts; - HeapCompareFn compFn; + HeapNode* min; + size_t nelts; + HeapCompareFn compFn; } Heap; - Heap* heapCreate(HeapCompareFn fn); -void heapDestroy(Heap *heap); +void heapDestroy(Heap* heap); HeapNode* heapMin(const Heap* heap); @@ -56,10 +56,10 @@ void heapRemove(Heap* heap, struct HeapNode* node); void heapDequeue(Heap* heap); -size_t heapSize(Heap *heap); +size_t heapSize(Heap* heap); #ifdef __cplusplus } #endif -#endif // TDENGINE_HASH_H +#endif /*_TD_UTIL_HEAP_H_*/ diff --git a/include/util/tidpool.h b/include/util/tidpool.h index 1a977fd04c27b14b3ce0ce242b3d523e1d67d688..8596b439e3f24d684bbabc4c7364f609119d224d 100644 --- a/include/util/tidpool.h +++ b/include/util/tidpool.h @@ -13,31 +13,34 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_IDPOOL_H -#define _TD_UTIL_IDPOOL_H +#ifndef _TD_UTIL_IDPOOL_H_ +#define _TD_UTIL_IDPOOL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -void *taosInitIdPool(int maxId); - -int taosUpdateIdPool(void *handle, int maxId); - -int taosIdPoolMaxSize(void *handle); - -int taosAllocateId(void *handle); - -void taosFreeId(void *handle, int id); - -void taosIdPoolCleanUp(void *handle); - -int taosIdPoolNumOfUsed(void *handle); - -bool taosIdPoolMarkStatus(void *handle, int id); +typedef struct { + int32_t maxId; + int32_t numOfFree; + int32_t freeSlot; + bool *freeList; + pthread_mutex_t mutex; +} id_pool_t; + +void *taosInitIdPool(int32_t maxId); +int32_t taosUpdateIdPool(id_pool_t *handle, int32_t maxId); +int32_t taosIdPoolMaxSize(id_pool_t *handle); +int32_t taosAllocateId(id_pool_t *handle); +void taosFreeId(id_pool_t *handle, int32_t id); +void taosIdPoolCleanUp(id_pool_t *handle); +int32_t taosIdPoolNumOfUsed(id_pool_t *handle); +bool taosIdPoolMarkStatus(id_pool_t *handle, int32_t id); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_IDPOOL_H*/ +#endif /*_TD_UTIL_IDPOOL_H_*/ diff --git a/include/util/tjson.h b/include/util/tjson.h index a0c2fef05b6cc365a469c38bd70acd8cf823eacf..2d9f433ab2a89fc2f4613a50a81eb3c3bfe613c9 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -16,35 +16,54 @@ #ifndef _TD_UTIL_JSON_H_ #define _TD_UTIL_JSON_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif -#include "os.h" - typedef void SJson; SJson* tjsonCreateObject(); -void tjsonDelete(SJson* pJson); - -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); +void tjsonDelete(SJson* pJson); +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number); int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number); +int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean); int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal); int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); +SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName); +int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal); +int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal); +int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal); +int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal); +int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal); +int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal); +int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal); +int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal); +int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal); +int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal); + +int32_t tjsonGetArraySize(const SJson* pJson); +SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index); + typedef int32_t (*FToJson)(const void* pObj, SJson* pJson); int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj); int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); -typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj); +typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); + +int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); +SJson* tjsonParse(const char* pStr); + #ifdef __cplusplus } #endif diff --git a/include/util/tkey.h b/include/util/tkey.h deleted file mode 100644 index 197cfff958416dc49db20b59dc8a2b6fe3216200..0000000000000000000000000000000000000000 --- a/include/util/tkey.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef _TD_UTIL_KEY_H -#define _TD_UTIL_KEY_H - -#ifdef __cplusplus -extern "C" { -#endif - -unsigned char *base64_decode(const char *value, int inlen, int *outlen); -char * base64_encode(const unsigned char *value, int vlen); -char * taosDesEncode(int64_t key, char *src, int len); -char * taosDesDecode(int64_t key, char *src, int len); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_KEY_H*/ \ No newline at end of file diff --git a/include/util/tlist.h b/include/util/tlist.h index 134873a993f3721e860ec443da281e2443dd04a1..caa642491898243a79509338447452bcd6dd31fa 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -12,8 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_LIST_H -#define _TD_UTIL_LIST_H + +#ifndef _TD_UTIL_LIST_H_ +#define _TD_UTIL_LIST_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -28,12 +31,12 @@ extern "C" { #define TD_SLIST(TYPE) \ struct { \ struct TYPE *sl_head_; \ - int sl_neles_; \ + int32_t sl_neles_; \ } -#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) -#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) -#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_HEAD(sl) ((sl)->sl_head_) +#define TD_SLIST_NELES(sl) ((sl)->sl_neles_) +#define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) #define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ @@ -79,16 +82,16 @@ extern "C" { struct { \ struct TYPE *dl_head_; \ struct TYPE *dl_tail_; \ - int dl_neles_; \ + int32_t dl_neles_; \ } -#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) -#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) +#define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) #define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) #define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) -#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) -#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) -#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) +#define TD_DLIST_HEAD(dl) ((dl)->dl_head_) +#define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) +#define TD_DLIST_NELES(dl) ((dl)->dl_neles_) #define TD_DLIST_INIT(dl) \ do { \ @@ -200,29 +203,29 @@ typedef struct SListNode { typedef struct { TD_DLIST(SListNode); - int eleSize; + int32_t eleSize; } SList; typedef struct { - SListNode * next; + SListNode *next; TD_LIST_DIRECTION_T direction; } SListIter; -#define listHead(l) TD_DLIST_HEAD(l) -#define listTail(l) TD_DLIST_TAIL(l) -#define listNEles(l) TD_DLIST_NELES(l) -#define listEleSize(l) ((l)->eleSize) -#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) +#define listHead(l) TD_DLIST_HEAD(l) +#define listTail(l) TD_DLIST_TAIL(l) +#define listNEles(l) TD_DLIST_NELES(l) +#define listEleSize(l) ((l)->eleSize) +#define isListEmpty(l) (TD_DLIST_NELES(l) == 0) #define listNodeFree(n) free(n) -void tdListInit(SList *list, int eleSize); +void tdListInit(SList *list, int32_t eleSize); void tdListEmpty(SList *list); -SList * tdListNew(int eleSize); -void * tdListFree(SList *list); +SList *tdListNew(int32_t eleSize); +void *tdListFree(SList *list); void tdListPrependNode(SList *list, SListNode *node); void tdListAppendNode(SList *list, SListNode *node); -int tdListPrepend(SList *list, void *data); -int tdListAppend(SList *list, void *data); +int32_t tdListPrepend(SList *list, void *data); +int32_t tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); SListNode *tdListGetHead(SList *list); @@ -239,4 +242,4 @@ SListNode *tdListNext(SListIter *pIter); } #endif -#endif /*_TD_UTIL_LIST_H*/ \ No newline at end of file +#endif /*_TD_UTIL_LIST_H_*/ \ No newline at end of file diff --git a/include/util/tlockfree.h b/include/util/tlockfree.h index fec346cd0290b914df038d3a965603b8b27b3fad..e49e89c309a99238b28535d9621ff20c93c66573 100644 --- a/include/util/tlockfree.h +++ b/include/util/tlockfree.h @@ -12,8 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOCK_FREE_H -#define _TD_UTIL_LOCK_FREE_H + +#ifndef _TD_UTIL_LOCK_FREE_H_ +#define _TD_UTIL_LOCK_FREE_H_ #include "os.h" @@ -22,7 +23,7 @@ extern "C" { #endif // reference counting -typedef void (*_ref_fn_t)(const void* pObj); +typedef void (*_ref_fn_t)(const void *pObj); #define T_REF_DECLARE() \ struct { \ @@ -67,8 +68,6 @@ typedef void (*_ref_fn_t)(const void* pObj); #define T_REF_VAL_GET(x) (x)->_ref.val - - // single writer multiple reader lock typedef volatile int32_t SRWLatch; @@ -78,35 +77,33 @@ void taosWUnLockLatch(SRWLatch *pLatch); void taosRLockLatch(SRWLatch *pLatch); void taosRUnLockLatch(SRWLatch *pLatch); - - // copy on read -#define taosCorBeginRead(x) for (uint32_t i_ = 1; 1; ++i_) { \ +#define taosCorBeginRead(x) \ + for (uint32_t i_ = 1; 1; ++i_) { \ int32_t old_ = atomic_add_fetch_32((x), 0); \ - if (old_ & 0x00000001) { \ - if (i_ % 1000 == 0) { \ - sched_yield(); \ - } \ - continue; \ + if (old_ & 0x00000001) { \ + if (i_ % 1000 == 0) { \ + sched_yield(); \ + } \ + continue; \ } -#define taosCorEndRead(x) \ - if (atomic_add_fetch_32((x), 0) == old_) { \ - break; \ - } \ +#define taosCorEndRead(x) \ + if (atomic_add_fetch_32((x), 0) == old_) { \ + break; \ + } \ } -#define taosCorBeginWrite(x) taosCorBeginRead(x) \ - if (atomic_val_compare_exchange_32((x), old_, old_ + 1) != old_) { \ - continue; \ - } +#define taosCorBeginWrite(x) \ + taosCorBeginRead(x) if (atomic_val_compare_exchange_32((x), old_, old_ + 1) != old_) { continue; } -#define taosCorEndWrite(x) atomic_add_fetch_32((x), 1); \ - break; \ +#define taosCorEndWrite(x) \ + atomic_add_fetch_32((x), 1); \ + break; \ } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_LOCK_FREE_H*/ +#endif /*_TD_UTIL_LOCK_FREE_H_*/ diff --git a/include/util/tlog.h b/include/util/tlog.h index 26a54173201c209c24461a35a9c4e462742543e0..6e6795e9a2c6c564e8d5dc4fe0bd642401907141 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOG_H -#define _TD_UTIL_LOG_H +#ifndef _TD_UTIL_LOG_H_ +#define _TD_UTIL_LOG_H_ #include "os.h" @@ -22,8 +22,7 @@ extern "C" { #endif -// log -extern int8_t tsAsyncLog; +extern bool tsAsyncLog; extern int32_t tsNumOfLogLines; extern int32_t tsLogKeepDays; extern int32_t dDebugFlag; @@ -32,9 +31,6 @@ extern int32_t mDebugFlag; extern int32_t cDebugFlag; extern int32_t jniDebugFlag; extern int32_t tmrDebugFlag; -extern int32_t httpDebugFlag; -extern int32_t mqttDebugFlag; -extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t qDebugFlag; @@ -42,23 +38,23 @@ extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; extern int32_t tqDebugFlag; -extern int32_t cqDebugFlag; -extern int32_t debugFlag; - -#define DEBUG_FATAL 1U -#define DEBUG_ERROR DEBUG_FATAL -#define DEBUG_WARN 2U -#define DEBUG_INFO DEBUG_WARN -#define DEBUG_DEBUG 4U -#define DEBUG_TRACE 8U -#define DEBUG_DUMP 16U +extern int32_t fsDebugFlag; +#define DEBUG_FATAL 1U +#define DEBUG_ERROR DEBUG_FATAL +#define DEBUG_WARN 2U +#define DEBUG_INFO DEBUG_WARN +#define DEBUG_DEBUG 4U +#define DEBUG_TRACE 8U +#define DEBUG_DUMP 16U #define DEBUG_SCREEN 64U -#define DEBUG_FILE 128U +#define DEBUG_FILE 128U -int32_t taosInitLog(char *logName, int32_t numOfLogLines, int32_t maxFiles); +int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); void taosResetLog(); +void taosSetAllDebugFlag(int32_t flag); +void taosDumpData(uint8_t *msg, int32_t len); void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) #ifdef __GNUC__ @@ -72,10 +68,20 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . #endif ; -void taosDumpData(unsigned char *msg, int32_t len); +extern int8_t tscEmbeddedInUtil; + +#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} +#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} +#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} + +#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } +#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_LOG_H*/ +#endif /*_TD_UTIL_LOG_H_*/ diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index 241647ba1eff40c2860368923008432768f92485..51906443f5ab874a2e7b16a11304ed6890f90437 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -13,14 +13,16 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_LOSERTREE_H -#define _TD_UTIL_LOSERTREE_H +#ifndef _TD_UTIL_LOSERTREE_H_ +#define _TD_UTIL_LOSERTREE_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param); +typedef int32_t (*__merge_compare_fn_t)(const void *, const void *, void *param); typedef struct STreeNode { int32_t index; @@ -31,16 +33,17 @@ typedef struct SMultiwayMergeTreeInfo { int32_t numOfSources; int32_t totalSources; __merge_compare_fn_t comparFn; - void * param; + void *param; struct STreeNode *pNode; } SMultiwayMergeTreeInfo; #define tMergeTreeGetChosenIndex(t_) ((t_)->pNode[0].index) #define tMergeTreeGetAdjustIndex(t_) (tMergeTreeGetChosenIndex(t_) + (t_)->numOfSources) -int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, + __merge_compare_fn_t compareFn); -void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree); +void tMergeTreeDestroy(SMultiwayMergeTreeInfo *pTree); void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); @@ -52,4 +55,4 @@ void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree); } #endif -#endif /*_TD_UTIL_LOSERTREE_H*/ +#endif /*_TD_UTIL_LOSERTREE_H_*/ diff --git a/include/util/tmacro.h b/include/util/tmacro.h index 297c37d62a99763462a7a61222ababf706a62b43..07c6e6509e4d5eedd7c14a830922b8347111f6e8 100644 --- a/include/util/tmacro.h +++ b/include/util/tmacro.h @@ -24,7 +24,7 @@ extern "C" { // Module init/clear MACRO definitions #define TD_MOD_UNINITIALIZED 0 -#define TD_MOD_INITIALIZED 1 +#define TD_MOD_INITIALIZED 1 typedef int8_t td_mode_flag_t; diff --git a/include/util/mallocator.h b/include/util/tmallocator.h similarity index 85% rename from include/util/mallocator.h rename to include/util/tmallocator.h index 5ecdc316a490aa1be7b57ec68d00b7653cebf864..e9eb3e1b727755fdcf34f5b9c78771362d6a0b6e 100644 --- a/include/util/mallocator.h +++ b/include/util/tmallocator.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MALLOCATOR_H_ -#define _TD_MALLOCATOR_H_ +#ifndef _TD_UTIL_MALLOCATOR_H_ +#define _TD_UTIL_MALLOCATOR_H_ #include "os.h" @@ -29,10 +29,10 @@ extern "C" { void (*free_)(struct TYPE *, void *ptr); \ } #define TD_MA_MALLOC_FUNC(TMA) (TMA)->malloc_ -#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ +#define TD_MA_FREE_FUNC(TMA) (TMA)->free_ #define TD_MA_MALLOC(TMA, SIZE) (*((TMA)->malloc_))(TMA, (SIZE)) -#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) +#define TD_MA_FREE(TMA, PTR) (*((TMA)->free_))(TMA, (PTR)) typedef struct SMemAllocator { void *impl; @@ -40,7 +40,7 @@ typedef struct SMemAllocator { } SMemAllocator; #define tMalloc(pMA, SIZE) TD_MA_MALLOC(PMA, SIZE) -#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR) +#define tFree(pMA, PTR) TD_MA_FREE(PMA, PTR) typedef struct SMemAllocatorFactory { void *impl; @@ -52,4 +52,4 @@ typedef struct SMemAllocatorFactory { } #endif -#endif /*_TD_MALLOCATOR_H_*/ \ No newline at end of file +#endif /*_TD_UTIL_MALLOCATOR_H_*/ \ No newline at end of file diff --git a/include/util/tmd5.h b/include/util/tmd5.h index f8114ad57bb1a863d5e580b3028439dbb86f9e0f..6f88d95e24195b5c5c77c53afad19c4503010c9b 100644 --- a/include/util/tmd5.h +++ b/include/util/tmd5.h @@ -25,6 +25,12 @@ #ifndef _TD_UTIL_MD5_H #define _TD_UTIL_MD5_H +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { uint32_t i[2]; /* number of _bits_ handled mod 2^64 */ uint32_t buf[4]; /* scratch buffer */ @@ -33,7 +39,11 @@ typedef struct { } T_MD5_CTX; void tMD5Init(T_MD5_CTX *mdContext); -void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen); +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, uint32_t inLen); void tMD5Final(T_MD5_CTX *mdContext); +#ifdef __cplusplus +} +#endif + #endif /*_TD_UTIL_MD5_H*/ diff --git a/include/util/tmempool.h b/include/util/tmempool.h index 3e3db738a9f372082d785bedb28da776c23ba59f..7a5aca7b34f6393c5440f030f3c416ca18b08a0f 100644 --- a/include/util/tmempool.h +++ b/include/util/tmempool.h @@ -12,25 +12,24 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#ifndef _TD_UTIL_MEMPOOL_H -#define _TD_UTIL_MEMPOOL_H +#ifndef _TD_UTIL_MEMPOOL_H_ +#define _TD_UTIL_MEMPOOL_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#define mpool_h void * - -mpool_h taosMemPoolInit(int maxNum, int blockSize); - -char *taosMemPoolMalloc(mpool_h handle); - -void taosMemPoolFree(mpool_h handle, char *p); +typedef void *mpool_h; -void taosMemPoolCleanUp(mpool_h handle); +mpool_h taosMemPoolInit(int32_t maxNum, int32_t blockSize); +char *taosMemPoolMalloc(mpool_h handle); +void taosMemPoolFree(mpool_h handle, char *p); +void taosMemPoolCleanUp(mpool_h handle); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_MEMPOOL_H*/ +#endif /*_TD_UTIL_MEMPOOL_H_*/ diff --git a/include/util/tnote.h b/include/util/tnote.h deleted file mode 100644 index e613ec7e4130312808e99dc5ed1f44298031ea1c..0000000000000000000000000000000000000000 --- a/include/util/tnote.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_NOTE_H -#define _TD_UTIL_NOTE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define MAX_NOTE_LINE_SIZE 66000 -#define NOTE_FILE_NAME_LEN 300 - -typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t fd; - int32_t openInProgress; - char name[NOTE_FILE_NAME_LEN]; - pthread_mutex_t mutex; -} SNoteObj; - -extern SNoteObj tsHttpNote; -extern SNoteObj tsTscNote; -extern SNoteObj tsInfoNote; - -int32_t taosInitNotes(); -void taosNotePrint(SNoteObj* pNote, const char* const format, ...); -void taosNotePrintBuffer(SNoteObj* pNote, char* buffer, int32_t len); - -#define nPrintHttp(...) \ - if (tsHttpEnableRecordSql) { \ - taosNotePrint(&tsHttpNote, __VA_ARGS__); \ - } - -#define nPrintTsc(...) \ - if (tsTscEnableRecordSql) { \ - taosNotePrint(&tsTscNote, __VA_ARGS__); \ - } - -#define nInfo(buffer, len) \ - if (tscEmbeddedInUtil == 1) { \ - taosNotePrintBuffer(&tsInfoNote, buffer, len); \ - } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_NOTE_H*/ diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index e989c31cd63f45cf0a9ac8c082f3cdb31f46436c..ce9a57c2c3ea22fc1c4b9ef9f8fbf00a73e137e5 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -13,27 +13,25 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TPAGEDBUF_H -#define TDENGINE_TPAGEDBUF_H +#ifndef _TD_UTIL_PAGEDBUF_H_ +#define _TD_UTIL_PAGEDBUF_H_ + +#include "thash.h" +#include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif -#include "tlist.h" -#include "thash.h" -#include "os.h" -#include "tlockfree.h" - -typedef struct SArray* SIDList; -typedef struct SPageInfo SPageInfo; +typedef struct SArray* SIDList; +typedef struct SPageInfo SPageInfo; typedef struct SDiskbasedBuf SDiskbasedBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes -#define DEFAULT_PAGE_SIZE (16384L) +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes typedef struct SFilePage { - int64_t num; + int32_t num; char data[]; } SFilePage; @@ -55,7 +53,7 @@ typedef struct SDiskbasedBufStatis { * @param handle * @return */ -int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); /** * @@ -64,7 +62,7 @@ int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t in * @param pageId * @return */ -SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId); +void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId); /** * @@ -80,7 +78,7 @@ SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId); * @param id * @return */ -SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id); +void* getBufPage(SDiskbasedBuf* pBuf, int32_t id); /** * release the referenced buf pages @@ -108,13 +106,13 @@ size_t getTotalBufSize(const SDiskbasedBuf* pBuf); * @param pBuf * @return */ -size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf); +size_t getNumOfBufGroupId(const SDiskbasedBuf* pBuf); /** * destroy result buffer * @param pBuf */ -void destroyResultBuf(SDiskbasedBuf* pBuf); +void destroyDiskbasedBuf(SDiskbasedBuf* pBuf); /** * @@ -137,6 +135,11 @@ int32_t getPageId(const SPageInfo* pPgInfo); */ int32_t getBufPageSize(const SDiskbasedBuf* pBuf); +/** + * + * @param pBuf + * @return + */ int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf); /** @@ -148,24 +151,45 @@ bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf); /** * Set the buffer page is dirty, and needs to be flushed to disk when swap out. - * @param pPageInfo + * @param pPage * @param dirty */ -void setBufPageDirty(SFilePage* pPageInfo, bool dirty); +void setBufPageDirty(void* pPage, bool dirty); + +/** + * Set the compress/ no-compress flag for paged buffer, when flushing data in disk. + * @param pBuf + */ +void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp); + +/** + * Set the pageId page buffer is not need + * @param pBuf + * @param pageId + */ +void dBufSetBufPageRecycled(SDiskbasedBuf *pBuf, void* pPage); /** * Print the statistics when closing this buffer * @param pBuf */ -void printStatisBeforeClose(SDiskbasedBuf* pBuf); +void dBufSetPrintInfo(SDiskbasedBuf* pBuf); /** - * return buf statistics. + * Return buf statistics. + * @param pBuf + * @return */ SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf); +/** + * Print the buffer statistics information + * @param pBuf + */ +void dBufPrintStatis(const SDiskbasedBuf* pBuf); + #ifdef __cplusplus } #endif -#endif // TDENGINE_TPAGEDBUF_H +#endif // _TD_UTIL_PAGEDBUF_H_ diff --git a/include/util/tqueue.h b/include/util/tqueue.h index cfa5a65c2a14543e6690fcf0b886d77696857cb0..d51184edfca2284f314ab9afe490d72430e8ccdf 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_QUEUE_H -#define _TD_UTIL_QUEUE_H +#ifndef _TD_UTIL_QUEUE_H_ +#define _TD_UTIL_QUEUE_H_ + #include "os.h" #ifdef __cplusplus @@ -47,7 +48,7 @@ typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); -void * taosAllocateQitem(int32_t size); +void *taosAllocateQitem(int32_t size); void taosFreeQitem(void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); @@ -80,4 +81,4 @@ int32_t taosGetQsetItemsNumber(STaosQset *qset); } #endif -#endif /*_TD_UTIL_QUEUE_H*/ +#endif /*_TD_UTIL_QUEUE_H_*/ diff --git a/include/util/tref.h b/include/util/tref.h index 6680204d63f58a75c61c45c340cc14c9129cd7c0..7e08bb045b08f56afe59436d07576b6e7604c12c 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -14,8 +14,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_REF_H -#define _TD_UTIL_REF_H +#ifndef _TD_UTIL_REF_H_ +#define _TD_UTIL_REF_H_ #include "os.h" @@ -25,45 +25,45 @@ extern "C" { // open a reference set, max is the mod used by hash, fp is the pointer to free resource function // return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately -int taosOpenRef(int max, void (*fp)(void *)); +int32_t taosOpenRef(int32_t max, void (*fp)(void *)); // close the reference set, refId is the return value by taosOpenRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosCloseRef(int refId); +int32_t taosCloseRef(int32_t refId); // add ref, p is the pointer to resource or pointer ID // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately -int64_t taosAddRef(int refId, void *p); +int64_t taosAddRef(int32_t refId, void *p); // remove ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosRemoveRef(int rsetId, int64_t rid); +int32_t taosRemoveRef(int32_t rsetId, int64_t rid); // acquire ref, rid is the reference ID returned by taosAddRef // return the resource p. On error, NULL is returned, and terrno is set appropriately -void *taosAcquireRef(int rsetId, int64_t rid); +void *taosAcquireRef(int32_t rsetId, int64_t rid); // release ref, rid is the reference ID returned by taosAddRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately -int taosReleaseRef(int rsetId, int64_t rid); +int32_t taosReleaseRef(int32_t rsetId, int64_t rid); -// return the first reference if rid is 0, otherwise return the next after current reference. +// return the first reference if rid is 0, otherwise return the next after current reference. // if return value is NULL, it means list is over(if terrno is set, it means error happens) -void *taosIterateRef(int rsetId, int64_t rid); +void *taosIterateRef(int32_t rsetId, int64_t rid); // return the number of references in system -int taosListRef(); +int32_t taosListRef(); #define RID_VALID(x) ((x) > 0) -/* sample code to iterate the refs +/* sample code to iterate the refs -void demoIterateRefs(int rsetId) { +void demoIterateRefs(int32_t rsetId) { void *p = taosIterateRef(refId, 0); while (p) { // process P - + // get the rid from p p = taosIterateRef(rsetId, rid); @@ -76,4 +76,4 @@ void demoIterateRefs(int rsetId) { } #endif -#endif /*_TD_UTIL_REF_H*/ +#endif /*_TD_UTIL_REF_H_*/ diff --git a/include/util/tsched.h b/include/util/tsched.h index d60c0c5ad4aa7a7e3e314d53765347b5209c8b84..3bf740f5285bc7559eea0f81f88462db40b28705 100644 --- a/include/util/tsched.h +++ b/include/util/tsched.h @@ -13,8 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_SCHED_H -#define _TD_UTIL_SCHED_H +#ifndef _TD_UTIL_SCHED_H_ +#define _TD_UTIL_SCHED_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -36,7 +38,7 @@ typedef struct SSchedMsg { * @param label the label of the queue * @return the created queue scheduler */ -void *taosInitScheduler(int capacity, int numOfThreads, const char *label); +void *taosInitScheduler(int32_t capacity, int32_t numOfThreads, const char *label); /** * Create a thread-safe ring-buffer based task queue and return the instance. @@ -47,7 +49,7 @@ void *taosInitScheduler(int capacity, int numOfThreads, const char *label); * @param tmrCtrl the timer controller, tmr_ctrl_t* * @return the created queue scheduler */ -void *taosInitSchedulerWithInfo(int capacity, int numOfThreads, const char *label, void *tmrCtrl); +void *taosInitSchedulerWithInfo(int32_t capacity, int32_t numOfThreads, const char *label, void *tmrCtrl); /** * Clean up the queue scheduler instance and free the memory. @@ -68,4 +70,4 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); } #endif -#endif /*_TD_UTIL_SCHED_H*/ +#endif /*_TD_UTIL_SCHED_H_*/ diff --git a/include/util/tskiplist.h b/include/util/tskiplist.h index 823a6f247f865786c0c92d0711863f598ed95abc..64cab08cfe87d430fded815878a5cad6083a4d81 100644 --- a/include/util/tskiplist.h +++ b/include/util/tskiplist.h @@ -16,22 +16,22 @@ #ifndef _TD_UTIL_SKILIST_H #define _TD_UTIL_SKILIST_H -#ifdef __cplusplus -extern "C" { -#endif - #include "os.h" #include "taos.h" #include "tarray.h" #include "tfunctional.h" -#define MAX_SKIP_LIST_LEVEL 15 +#ifdef __cplusplus +extern "C" { +#endif + +#define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 // For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) #define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update!=0 case) // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 @@ -39,17 +39,17 @@ extern "C" { typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); -typedef void (*sl_patch_row_fn_t)(void * pDst, const void * pSrc); -typedef void* (*iter_next_fn_t)(void *iter); +typedef void (*sl_patch_row_fn_t)(void *pDst, const void *pSrc); +typedef void *(*iter_next_fn_t)(void *iter); typedef struct SSkipListNode { - uint8_t level; - void * pData; + uint8_t level; + void *pData; struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_GET_NODE_DATA(n) (n)->pData -#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] /* @@ -100,14 +100,10 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; -typedef enum { - SSkipListPutSuccess = 0, - SSkipListPutEarlyStop = 1, - SSkipListPutSkipOne = 2 -} SSkipListPutStatus; +typedef enum { SSkipListPutSuccess = 0, SSkipListPutEarlyStop = 1, SSkipListPutSkipOne = 2 } SSkipListPutStatus; typedef struct SSkipList { - unsigned int seed; + uint32_t seed; __compar_fn_t comparFn; __sl_key_fn_t keyFn; pthread_rwlock_t *lock; @@ -117,41 +113,41 @@ typedef struct SSkipList { uint8_t type; // static info above uint8_t level; uint32_t size; - SSkipListNode * pHead; // point to the first element - SSkipListNode * pTail; // point to the last element + SSkipListNode *pHead; // point to the first element + SSkipListNode *pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif - tGenericSavedFunc* insertHandleFn; + tGenericSavedFunc *insertHandleFn; } SSkipList; typedef struct SSkipListIterator { - SSkipList * pSkipList; + SSkipList *pSkipList; SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator - SSkipListNode *next; // next points to the true qualified node in skiplist + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator + SSkipListNode *next; // next points to the true qualified node in skiplist } SSkipListIterator; -#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) -#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) #define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) -#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) -#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) -#define SL_SIZE(s) (s)->size +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, __sl_key_fn_t fn); void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t iterate); -SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); bool tSkipListIterNext(SSkipListIterator *iter); -SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); -void * tSkipListDestroyIter(SSkipListIterator *iter); +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); +void *tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); @@ -159,4 +155,4 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNod } #endif -#endif /*_TD_UTIL_SKILIST_H*/ +#endif /*_TD_UTIL_SKILIST_H*/ diff --git a/include/util/tstrbuild.h b/include/util/tstrbuild.h index 48a302531cf9c80cc9c8b4ceff3b4ba714127dd1..89f3472346276ca749176c048d9ec118c4b99d19 100644 --- a/include/util/tstrbuild.h +++ b/include/util/tstrbuild.h @@ -13,12 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_STRING_BUILDER_H -#define _TD_UTIL_STRING_BUILDER_H +#ifndef _TD_UTIL_STRING_BUILDER_H_ +#define _TD_UTIL_STRING_BUILDER_H_ -#include -#include -#include +#include "os.h" #ifdef __cplusplus extern "C" { @@ -26,16 +24,16 @@ extern "C" { typedef struct SStringBuilder { jmp_buf jb; - size_t size; - size_t pos; - char* buf; + size_t size; + size_t pos; + char* buf; } SStringBuilder; #define taosStringBuilderSetJmp(sb) setjmp((sb)->jb) -void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size); +void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size); char* taosStringBuilderGetResult(SStringBuilder* sb, size_t* len); -void taosStringBuilderDestroy(SStringBuilder* sb); +void taosStringBuilderDestroy(SStringBuilder* sb); void taosStringBuilderAppend(SStringBuilder* sb, const void* data, size_t len); void taosStringBuilderAppendChar(SStringBuilder* sb, char c); @@ -49,4 +47,4 @@ void taosStringBuilderAppendDouble(SStringBuilder* sb, double v); } #endif -#endif /*_TD_UTIL_STRING_BUILDER_H*/ \ No newline at end of file +#endif /*_TD_UTIL_STRING_BUILDER_H_*/ \ No newline at end of file diff --git a/include/util/tthread.h b/include/util/tthread.h index 7a5fd1f4c877024f2e3b8ade79a4094c1ef91ebc..49412069445d0f1cb89631683daa6411ca88879d 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -13,25 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_THREAD_H -#define _TD_UTIL_THREAD_H +#ifndef _TD_UTIL_THREAD_H_ +#define _TD_UTIL_THREAD_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -#include "os.h" -#include "tdef.h" - -// create new thread pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); -// destory thread -bool taosDestoryThread(pthread_t* pthread); -// thread running return true -bool taosThreadRunning(pthread_t* pthread); +bool taosDestoryThread(pthread_t* pthread); +bool taosThreadRunning(pthread_t* pthread); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_THREAD_H*/ +#endif /*_TD_UTIL_THREAD_H_*/ diff --git a/include/util/ttimer.h b/include/util/ttimer.h index 89ec6cd8d9d000bd98effc8804c52be8ce48c740..f2ee825c4e5ae4ab8b48184e78e89b5c99cf6ff6 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_TIMER_H -#define _TD_UTIL_TIMER_H +#ifndef _TD_UTIL_TIMER_H_ +#define _TD_UTIL_TIMER_H_ #include "os.h" @@ -25,24 +25,23 @@ extern "C" { typedef void *tmr_h; typedef void (*TAOS_TMR_CALLBACK)(void *, void *); -extern int taosTmrThreads; -extern uint32_t tsMaxTmrCtrl; +extern int32_t taosTmrThreads; #define MSECONDS_PER_TICK 5 -void *taosTmrInit(int maxTmr, int resoultion, int longest, const char *label); +void *taosTmrInit(int32_t maxTmr, int32_t resoultion, int32_t longest, const char *label); -tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void *param, void *handle); +tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *handle); bool taosTmrStop(tmr_h tmrId); bool taosTmrStopA(tmr_h *timerId); -bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void *param, void *handle, tmr_h *pTmrId); +bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *handle, tmr_h *pTmrId); void taosTmrCleanUp(void *handle); -int32_t taosInitTimer(void (*callback)(int), int32_t ms); +int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); void taosUninitTimer(); @@ -50,4 +49,4 @@ void taosUninitTimer(); } #endif -#endif /*_TD_UTIL_TIMER_H*/ +#endif /*_TD_UTIL_TIMER_H_*/ diff --git a/include/util/tutil.h b/include/util/tutil.h index c315948702af31b5dbcf8cde71331f132554532f..e0f92be76a90d5abbfa22366d8513f3e06facbbd 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -13,18 +13,18 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_UTIL_H -#define _TD_UTIL_UTIL_H - -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _TD_UTIL_UTIL_H_ +#define _TD_UTIL_UTIL_H_ #include "os.h" #include "tcrc32c.h" #include "tdef.h" #include "tmd5.h" +#ifdef __cplusplus +extern "C" { +#endif + int32_t strdequote(char *src); int32_t strndequote(char *dst, const char *z, int32_t len); int32_t strRmquote(char *z, int32_t len); @@ -49,7 +49,7 @@ void taosIpPort2String(uint32_t ip, uint16_t port, char *str); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, inBuf, (unsigned int)inLen); + tMD5Update(&context, inBuf, (uint32_t)inLen); tMD5Final(&context); memcpy(target, context.digest, tListLen(context.digest)); } @@ -57,17 +57,17 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *tar static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *target) { T_MD5_CTX context; tMD5Init(&context); - tMD5Update(&context, inBuf, (unsigned int)len); + tMD5Update(&context, inBuf, (uint32_t)len); tMD5Final(&context); - sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], - context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], - context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], - context.digest[13], context.digest[14], context.digest[15]); + sprintf(target, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], + context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], + context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); } #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_UTIL_H*/ +#endif /*_TD_UTIL_UTIL_H_*/ diff --git a/include/util/tversion.h b/include/util/tversion.h index 3d7a7e1b665f219eee4d9f94a7dd56bc1d0d5bcc..c924752a01f52f01f2cb40dbc6e16770f23d1ea2 100644 --- a/include/util/tversion.h +++ b/include/util/tversion.h @@ -13,8 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_VERSION_H -#define _TD_UTIL_VERSION_H +#ifndef _TD_UTIL_VERSION_H_ +#define _TD_UTIL_VERSION_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { @@ -28,4 +30,4 @@ int32_t taosCheckVersionCompatible(int32_t clientVer, int32_t serverVer, int32_t } #endif -#endif /*_TD_UTIL_VERSION_H*/ +#endif /*_TD_UTIL_VERSION_H_*/ diff --git a/include/util/tworker.h b/include/util/tworker.h index 771c7c943359b08e664f3d7de45fe6a4ecf6ada3..e6f6bc077c69f05498df2536c7de55a488dd8f93 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_WORKER_H -#define _TD_UTIL_WORKER_H +#ifndef _TD_UTIL_WORKER_H_ +#define _TD_UTIL_WORKER_H_ + #include "tqueue.h" #ifdef __cplusplus @@ -34,25 +35,25 @@ typedef struct SQWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - STaosQset * qset; - const char * name; - SQWorker * workers; + STaosQset *qset; + const char *name; + SQWorker *workers; pthread_mutex_t mutex; } SQWorkerPool, SFWorkerPool; typedef struct SWWorker { int32_t id; // worker id pthread_t thread; // thread - STaosQall * qall; - STaosQset * qset; // queue set + STaosQall *qall; + STaosQset *qset; // queue set SWWorkerPool *pool; } SWWorker; typedef struct SWWorkerPool { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic - const char * name; - SWWorker * workers; + const char *name; + SWWorker *workers; pthread_mutex_t mutex; } SWWorkerPool; @@ -75,4 +76,4 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); } #endif -#endif /*_TD_UTIL_WORKER_H*/ +#endif /*_TD_UTIL_WORKER_H_*/ diff --git a/include/util/types.h b/include/util/types.h index 25688df56cc0276931419cf24cce5b19ed690d77..f7a535c965fd6deac8dc2eb1d486c072a0e867e6 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -22,14 +22,14 @@ extern "C" { #endif -#define GET_INT8_VAL(x) (*(int8_t *)(x)) -#define GET_INT16_VAL(x) (*(int16_t *)(x)) -#define GET_INT32_VAL(x) (*(int32_t *)(x)) -#define GET_INT64_VAL(x) (*(int64_t *)(x)) -#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) -#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) -#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) -#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t *)(x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) static FORCE_INLINE float taos_align_get_float(const char *pBuf) { #if __STDC_VERSION__ >= 201112L @@ -64,23 +64,33 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) { // #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } // #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } // #else - #define GET_FLOAT_VAL(x) (*(float *)(x)) - #define GET_DOUBLE_VAL(x) (*(double *)(x)) - #define SET_BIGINT_VAL(x, y) { (*(int64_t *)(x)) = (int64_t)(y); } - #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } - #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } - #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } - #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +#define GET_FLOAT_VAL(x) (*(float *)(x)) +#define GET_DOUBLE_VAL(x) (*(double *)(x)) +#define SET_BIGINT_VAL(x, y) \ + { (*(int64_t *)(x)) = (int64_t)(y); } +#define SET_FLOAT_VAL(x, y) \ + { (*(float *)(x)) = (float)(y); } +#define SET_DOUBLE_VAL(x, y) \ + { (*(double *)(x)) = (double)(y); } +#define SET_FLOAT_PTR(x, y) \ + { (*(float *)(x)) = (*(float *)(y)); } +#define SET_DOUBLE_PTR(x, y) \ + { (*(double *)(x)) = (*(double *)(y)); } // #endif -typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 -#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) +typedef uint16_t VarDataLenT; // maxVarDataLen: 32767 +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) #define varDataLen(v) ((VarDataLenT *)(v))[0] #define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) typedef int32_t VarDataOffsetT; +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + #ifdef __cplusplus } #endif diff --git a/include/util/ulog.h b/include/util/ulog.h deleted file mode 100644 index 89d9f8947614701e67757ec9fb25b57c94c5a057..0000000000000000000000000000000000000000 --- a/include/util/ulog.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_ULOG_H -#define _TD_UTIL_ULOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "tlog.h" - -extern int32_t uDebugFlag; -extern int8_t tscEmbeddedInUtil; - -#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uInfo(...) { if (uDebugFlag & DEBUG_INFO) { taosPrintLog("UTL ", tscEmbeddedInUtil ? 255 : uDebugFlag, __VA_ARGS__); }} -#define uDebug(...) { if (uDebugFlag & DEBUG_DEBUG) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} -#define uTrace(...) { if (uDebugFlag & DEBUG_TRACE) { taosPrintLog("UTL ", uDebugFlag, __VA_ARGS__); }} - -#define pError(...) { taosPrintLog("APP ERROR ", 255, __VA_ARGS__); } -#define pPrint(...) { taosPrintLog("APP ", 255, __VA_ARGS__); } - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_ULOG_H*/ diff --git a/include/util/version.h b/include/util/version.h new file mode 100644 index 0000000000000000000000000000000000000000..b241dd248b9a673591bbc94e1a71a78ad721919e --- /dev/null +++ b/include/util/version.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_VERSION_H_ +#define _TD_UTIL_VERSION_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern char version[]; +extern char compatible_version[]; +extern char gitinfo[]; +extern char gitinfoOfInternal[]; +extern char buildinfo[]; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_VERSION_H_*/ diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d7574466cdfbe15ea38145c7dd3a9d0a3895e9bb..79909a569606758f9adc33a036881830fab6ffa7 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,17 +20,19 @@ extern "C" { #endif +#include "tcommon.h" +#include "parser.h" +#include "query.h" #include "taos.h" -#include "common.h" -#include "tmsg.h" #include "tdef.h" #include "tep.h" #include "thash.h" #include "tlist.h" +#include "tmsg.h" #include "tmsgtype.h" #include "trpc.h" -#include "query.h" -#include "parser.h" + +#include "tconfig.h" #define CHECK_CODE_GOTO(expr, label) \ do { \ @@ -46,12 +48,12 @@ extern "C" { typedef struct SAppInstInfo SAppInstInfo; typedef struct SHbConnInfo { - void *param; - SClientHbReq *req; + void* param; + SClientHbReq* req; } SHbConnInfo; typedef struct SAppHbMgr { - char *key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -62,15 +64,13 @@ typedef struct SAppHbMgr { // connection SAppInstInfo* pAppInstInfo; // info - SHashObj* activeInfo; // hash - SHashObj* connInfo; // hash + SHashObj* activeInfo; // hash + SHashObj* connInfo; // hash } SAppHbMgr; +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); -typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); - -typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); - +typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct SClientHbMgr { int8_t inited; @@ -83,63 +83,62 @@ typedef struct SClientHbMgr { FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; - typedef struct SQueryExecMetric { - int64_t start; // start timestamp - int64_t parsed; // start to parse - int64_t send; // start to send to server - int64_t rsp; // receive response from server + int64_t start; // start timestamp + int64_t parsed; // start to parse + int64_t send; // start to send to server + int64_t rsp; // receive response from server } SQueryExecMetric; typedef struct SInstanceSummary { - uint64_t numOfInsertsReq; - uint64_t numOfInsertRows; - uint64_t insertElapsedTime; - uint64_t insertBytes; // submit to tsdb since launched. - - uint64_t fetchBytes; - uint64_t queryElapsedTime; - uint64_t numOfSlowQueries; - uint64_t totalRequests; - uint64_t currentRequests; // the number of SRequestObj + uint64_t numOfInsertsReq; + uint64_t numOfInsertRows; + uint64_t insertElapsedTime; + uint64_t insertBytes; // submit to tsdb since launched. + + uint64_t fetchBytes; + uint64_t queryElapsedTime; + uint64_t numOfSlowQueries; + uint64_t totalRequests; + uint64_t currentRequests; // the number of SRequestObj } SInstanceSummary; typedef struct SHeartBeatInfo { - void *pTimer; // timer, used to send request msg to mnode + void* pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; - SList *pConnList; // STscObj linked list - int64_t clusterId; - void *pTransporter; - struct SAppHbMgr *pAppHbMgr; + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; + SList* pConnList; // STscObj linked list + int64_t clusterId; + void* pTransporter; + struct SAppHbMgr* pAppHbMgr; }; typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char *ep; + char* ep; int32_t pid; int32_t numOfThreads; - SHashObj *pInstMap; + SHashObj* pInstMap; pthread_mutex_t mutex; } SAppInfo; typedef struct STscObj { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char db[TSDB_DB_FNAME_LEN]; - char ver[128]; - int32_t acctId; - uint32_t connId; - int32_t connType; - uint64_t id; // ref ID returned by taosAddRef - pthread_mutex_t mutex; // used to protect the operation on db - int32_t numOfReqs; // number of sqlObj bound to this connection - SAppInstInfo *pAppInfo; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char ver[128]; + int32_t acctId; + uint32_t connId; + int32_t connType; + uint64_t id; // ref ID returned by taosAddRef + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj bound to this connection + SAppInstInfo* pAppInfo; } STscObj; typedef struct SMqConsumer { @@ -147,49 +146,49 @@ typedef struct SMqConsumer { } SMqConsumer; typedef struct SReqResultInfo { - const char *pRspMsg; - const char *pData; - TAOS_FIELD *fields; - uint32_t numOfCols; - int32_t *length; - TAOS_ROW row; - char **pCol; - uint32_t numOfRows; - uint64_t totalRows; - uint32_t current; - bool completed; + const char* pRspMsg; + const char* pData; + TAOS_FIELD* fields; + uint32_t numOfCols; + int32_t* length; + TAOS_ROW row; + char** pCol; + uint32_t numOfRows; + uint64_t totalRows; + uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { - int64_t execId; // showId/queryId - int32_t vgId; - SArray *pArray; // SArray - int32_t currentIndex; // current accessed vgroup index. + int64_t execId; // showId/queryId + int32_t vgId; + SArray* pArray; // SArray + int32_t currentIndex; // current accessed vgroup index. } SShowReqInfo; typedef struct SRequestSendRecvBody { - tsem_t rspSem; // not used now + tsem_t rspSem; // not used now void* fp; - SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. SDataBuf requestMsg; - struct SSchJob *pQueryJob; // query job, created according to sql query DAG. - struct SQueryDag *pDag; // the query dag, generated according to the sql statement. + struct SSchJob* pQueryJob; // query job, created according to sql query DAG. + struct SQueryDag* pDag; // the query dag, generated according to the sql statement. SReqResultInfo resInfo; } SRequestSendRecvBody; -#define ERROR_MSG_BUF_DEFAULT_SIZE 512 +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 typedef struct SRequestObj { - uint64_t requestId; - int32_t type; // request type - STscObj *pTscObj; - char *sqlstr; // sql string - int32_t sqlLen; - int64_t self; - char *msgBuf; - void *pInfo; // sql parse info, generated by parser module - int32_t code; - SQueryExecMetric metric; + uint64_t requestId; + int32_t type; // request type + STscObj* pTscObj; + char* sqlstr; // sql string + int32_t sqlLen; + int64_t self; + char* msgBuf; + void* pInfo; // sql parse info, generated by parser module + int32_t code; + SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -198,51 +197,52 @@ extern int32_t clientReqRefPool; extern int32_t clientConnRefPool; extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); +int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); -int taos_init(); +int taos_init(); -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo); -void destroyTscObj(void*pObj); +void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); +void destroyTscObj(void* pObj); uint64_t generateRequestId(); -void *createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); +void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -char *getDbOfConnection(STscObj* pObj); +char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); void taos_init_imp(void); -int taos_options_imp(TSDB_OPTION option, const char *str); +int taos_options_imp(TSDB_OPTION option, const char* str); -void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); +void* openTransporter(const char* user, const char* auth, int32_t numOfThreads); +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); - -void *doFetchRow(SRequestObj* pRequest); +TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, + uint16_t port); -void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +void* doFetchRow(SRequestObj* pRequest); +void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); -int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); +int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); -// --- heartbeat +// --- heartbeat // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(void); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); @@ -253,7 +253,6 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v // --- mq void hbMgrInitMqHbRspHandle(); - #ifdef __cplusplus } #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cb73701bfa389ede7ed8073728acd7d418a369c5..08285c9d26f923c7ebfb420e7221c182e728313d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,33 +13,30 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" #include "clientInt.h" #include "clientLog.h" +#include "os.h" #include "query.h" #include "scheduler.h" -#include "tmsg.h" #include "tcache.h" -#include "tconfig.h" #include "tglobal.h" -#include "tnote.h" +#include "tmsg.h" #include "tref.h" #include "trpc.h" #include "ttime.h" -#include "ttimezone.h" #define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 +#define TSC_VAR_RELEASED 0 -SAppInfo appInfo; -int32_t clientReqRefPool = -1; -int32_t clientConnRefPool = -1; +SAppInfo appInfo; +int32_t clientReqRefPool = -1; +int32_t clientConnRefPool = -1; static pthread_once_t tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj* pRequest) { +static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = (STscObj *)taosAcquireRef(clientConnRefPool, pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -53,60 +50,49 @@ static void registerRequest(SRequestObj* pRequest) { int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); - tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%"PRIx64, pRequest->self, - pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 + ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } -static void deregisterRequest(SRequestObj* pRequest) { +static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj* pTscObj = pRequest->pTscObj; - SInstanceSummary* pActivity = &pTscObj->pAppInfo->summary; + STscObj * pTscObj = pRequest->pTscObj; + SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; - tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", reqId:0x%"PRIx64" elapsed:%"PRIu64" ms, current:%d, app current:%d", pRequest->self, pTscObj->id, - pRequest->requestId, duration, num, currentInst); + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 + " ms, current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration, num, currentInst); taosReleaseRef(clientConnRefPool, pTscObj->id); } -static void tscInitLogFile() { - taosReadGlobalLogCfg(); - if (mkdir(tsLogDir, 0755) != 0 && errno != EEXIST) { - printf("failed to create log dir:%s\n", tsLogDir); - } - - const char *defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); - } -} // todo close the transporter properly -void closeTransporter(STscObj* pTscObj) { +void closeTransporter(STscObj *pTscObj) { if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%"PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } // TODO refactor -void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { +void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "TSC"; rpcInit.numOfThreads = numOfThread; rpcInit.cfp = processMsgFromServer; + rpcInit.pfp = persistConnForSpecificMsg; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)user; @@ -115,7 +101,7 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { rpcInit.spi = 1; rpcInit.secret = (char *)auth; - void* pDnodeConn = rpcOpen(&rpcInit); + void *pDnodeConn = rpcOpen(&rpcInit); if (pDnodeConn == NULL) { tscError("failed to init connection to server"); return NULL; @@ -130,12 +116,12 @@ void destroyTscObj(void *pObj) { SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } -void* createTscObj(const char* user, const char* auth, const char *db, SAppInstInfo* pAppInfo) { +void *createTscObj(const char *user, const char *auth, const char *db, SAppInstInfo *pAppInfo) { STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -153,11 +139,11 @@ void* createTscObj(const char* user, const char* auth, const char *db, SAppInstI pthread_mutex_init(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); - tscDebug("connObj created, 0x%"PRIx64, pObj->id); + tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; } -void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type) { +void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); SRequestObj *pRequest = (SRequestObj *)calloc(1, sizeof(SRequestObj)); @@ -166,20 +152,20 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty return NULL; } - pRequest->requestId = generateRequestId(); + pRequest->requestId = generateRequestId(); pRequest->metric.start = taosGetTimestampMs(); - pRequest->type = type; - pRequest->pTscObj = pObj; - pRequest->body.fp = fp; // not used it yet - pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); + pRequest->type = type; + pRequest->pTscObj = pObj; + pRequest->body.fp = fp; // not used it yet + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { +static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { tfree(pResInfo->pRspMsg); tfree(pResInfo->length); tfree(pResInfo->row); @@ -187,9 +173,9 @@ static void doFreeReqResultInfo(SReqResultInfo* pResInfo) { tfree(pResInfo->fields); } -static void doDestroyRequest(void* p) { +static void doDestroyRequest(void *p) { assert(p != NULL); - SRequestObj* pRequest = (SRequestObj*)p; + SRequestObj *pRequest = (SRequestObj *)p; assert(RID_VALID(pRequest->self)); @@ -208,7 +194,7 @@ static void doDestroyRequest(void* p) { tfree(pRequest); } -void destroyRequest(SRequestObj* pRequest) { +void destroyRequest(SRequestObj *pRequest) { if (pRequest == NULL) { return; } @@ -225,16 +211,17 @@ void taos_init_imp(void) { srand(taosGetTimestampSec()); deltaToUtcInitOnce(); - taosInitGlobalCfg(); - taosReadCfgFromFile(); - tscInitLogFile(); - if (taosCheckAndPrintCfg()) { + if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) { + tscInitRes = -1; + return; + } + + if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } - taosInitNotes(); initMsgHandleFp(); initQueryModuleMsgHandle(); @@ -245,21 +232,21 @@ void taos_init_imp(void) { SSchedulerCfg scfg = {.maxJobNum = 100}; schedulerInit(&scfg); - tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp); + tscDebug("starting to initialize TAOS driver"); taosSetCoreDump(true); initTaskQueue(); clientConnRefPool = taosOpenRef(200, destroyTscObj); - clientReqRefPool = taosOpenRef(40960, doDestroyRequest); + clientReqRefPool = taosOpenRef(40960, doDestroyRequest); taosGetAppName(appInfo.appName, NULL); pthread_mutex_init(&appInfo.mutex, NULL); - appInfo.pid = taosGetPId(); + appInfo.pid = taosGetPId(); appInfo.startTime = taosGetTimestampMs(); - appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + appInfo.pInstMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tscDebug("client is initialized successfully"); } @@ -269,6 +256,7 @@ int taos_init() { } int taos_options_imp(TSDB_OPTION option, const char *str) { +#if 0 SGlobalCfg *cfg = NULL; switch (option) { @@ -281,7 +269,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set config file directory:%s", str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -296,7 +285,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscInfo("set shellActivityTimer:%d", tsShellActivityTimer); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], *(int32_t *)cfg->ptr); } break; @@ -305,7 +295,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("Invalid locale:%s, use default", str); return -1; } @@ -313,8 +303,8 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { char sep = '.'; - if (strlen(tsLocale) == 0) { // locale does not set yet - char* defaultLocale = setlocale(LC_CTYPE, ""); + if (strlen(tsLocale) == 0) { // locale does not set yet + char *defaultLocale = setlocale(LC_CTYPE, ""); // The locale of the current OS does not be set correctly, so the default locale cannot be acquired. // The launch of current system will abort soon. @@ -323,21 +313,21 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return -1; } - tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, defaultLocale, TD_LOCALE_LEN); } // set the user specified locale char *locale = setlocale(LC_CTYPE, str); - if (locale != NULL) { // failed to set the user specified locale + if (locale != NULL) { // failed to set the user specified locale tscInfo("locale set, prev locale:%s, new locale:%s", tsLocale, locale); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; - } else { // set the user specified locale failed, use default LC_CTYPE as current locale + } else { // set the user specified locale failed, use default LC_CTYPE as current locale locale = setlocale(LC_CTYPE, tsLocale); tscInfo("failed to set locale:%s, current locale:%s", str, tsLocale); } - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); + tstrncpy(tsLocale, locale, TD_LOCALE_LEN); char *charset = strrchr(tsLocale, sep); if (charset != NULL) { @@ -352,7 +342,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, charset); } - tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, charset, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { @@ -360,11 +350,12 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } free(charset); - } else { // it may be windows system + } else { // it may be windows system tscInfo("charset remains:%s", tsCharset); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; } @@ -375,7 +366,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); size_t len = strlen(str); - if (len == 0 || len > TSDB_LOCALE_LEN) { + if (len == 0 || len > TD_LOCALE_LEN) { tscInfo("failed to set charset:%s", str); return -1; } @@ -388,13 +379,14 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscInfo("charset changed from %s to %s", tsCharset, str); } - tstrncpy(tsCharset, str, TSDB_LOCALE_LEN); + tstrncpy(tsCharset, str, TD_LOCALE_LEN); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; } else { tscInfo("charset:%s not valid", str); } } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -405,12 +397,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { assert(cfg != NULL); if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) { - tstrncpy(tsTimezone, str, TSDB_TIMEZONE_LEN); + tstrncpy(tsTimezone, str, TD_TIMEZONE_LEN); tsSetTimeZone(); cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION; tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, str); } else { - tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); + tscWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, str, + tsCfgStatusStr[cfg->cfgStatus], (char *)cfg->ptr); } break; @@ -419,7 +412,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { tscError("Invalid option %d", option); return -1; } - +#endif return 0; } @@ -434,7 +427,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static int32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -448,9 +441,9 @@ uint64_t generateRequestId() { } } - int64_t ts = taosGetTimestampMs(); - uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + int64_t ts = taosGetTimestampMs(); + uint64_t pid = taosGetPId(); + int32_t val = atomic_add_fetch_32(&requestSerialId, 1); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); return id; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2eff353c3947d2fa7f908300b79feadd0c75f93c..3c6a49c3db25cbcc4a7b5f76d6081728a5a6f129 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -8,7 +8,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tref.h" @@ -227,23 +226,27 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; - if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { - SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pTransporter, NULL, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); - if (code != TSDB_CODE_SUCCESS) { - // handle error and retry - } else { - if (pRequest->body.pQueryJob != NULL) { - schedulerFreeJob(pRequest->body.pQueryJob); - } + SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; + int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); + if (code != TSDB_CODE_SUCCESS) { + if (pRequest->body.pQueryJob != NULL) { + schedulerFreeJob(pRequest->body.pQueryJob); } - pRequest->body.resInfo.numOfRows = res.numOfRows; - pRequest->code = res.code; + pRequest->code = code; return pRequest->code; } - return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); + if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { + pRequest->body.resInfo.numOfRows = res.numOfRows; + + if (pRequest->body.pQueryJob != NULL) { + schedulerFreeJob(pRequest->body.pQueryJob); + } + } + + pRequest->code = res.code; + return pRequest->code; } TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { @@ -254,11 +257,9 @@ TAOS_RES* taos_query_l(TAOS* taos, const char* sql, int sqlLen) { return NULL; } - nPrintTsc("%s", sql) - - SRequestObj* pRequest = NULL; - SQueryNode* pQueryNode = NULL; - SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr)); terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -370,7 +371,6 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; pMsgSendInfo->param = pRequest; - SConnectReq connectReq = {0}; STscObj* pObj = pRequest->pTscObj; @@ -398,7 +398,9 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody->msgInfo.pData); tfree(pMsgBody); } - +bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType) { + return msgType == TDMT_VND_QUERY_RSP || msgType == TDMT_VND_FETCH_RSP || msgType == TDMT_VND_RES_READY_RSP; +} void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo* pSendInfo = (SMsgSendInfo*)pMsg->ahandle; assert(pMsg->ahandle != NULL); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index b29813be03b3163d0399133c2d12c658eda3ccc5..5bcc287116d2de8a9f713f26cfdbed74c7a52fd9 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -7,6 +7,7 @@ #include "tmsg.h" #include "tglobal.h" #include "catalog.h" +#include "version.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -56,9 +57,7 @@ void taos_cleanup(void) { } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0) ? port : tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, port, user, db); if (user == NULL) { user = TSDB_DEFAULT_USER; } @@ -67,7 +66,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - return taos_connect_internal(ip, user, pass, NULL, db, p); + return taos_connect_internal(ip, user, pass, NULL, db, port); } void taos_close(TAOS* taos) { diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 5b4afda92326da4fed8d332a6febb1d1c64bab65..f31130acf44b6cd74ce0e3f82923f1ffa805e4eb 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -24,7 +24,6 @@ #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" -#include "tnote.h" #include "tpagedbuf.h" #include "tqueue.h" #include "tref.h" diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 5414f17a6247769ba38c84d0dac8c37d8e25168e..7bba0935882bcac5b784dee40a6b4b8f24bc6ae9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -49,12 +49,14 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } -#if 0 +#if 1 TEST(testCase, connect_Test) { +// taos_options(TSDB_OPTION_CONFIGDIR, "/home/ubuntu/first/cfg"); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); @@ -332,7 +334,7 @@ TEST(testCase, create_ctable_Test) { } taos_free_result(pRes); - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + pRes = taos_query(pConn, "create table tu using sts tags('2021-10-10 1:1:1');"); if (taos_errno(pRes) != 0) { printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); } diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp index 9f8ff7143a822cfe3abb69b1475f884397501436..c34c10d8714fa443bb161333ab622093ec6eebd7 100644 --- a/source/client/test/tmqTest.cpp +++ b/source/client/test/tmqTest.cpp @@ -34,7 +34,7 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { - taosInitGlobalCfg(); + // taosInitGlobalCfg(); // taos_init(); } diff --git a/source/common/src/tcompare.c b/source/common/src/tcompare.c deleted file mode 100644 index ef441c97c786bc51e4db5cfdcac2bc6c5abf5f77..0000000000000000000000000000000000000000 --- a/source/common/src/tcompare.c +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tcompare.h" - - -__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { - __compar_fn_t comparFn = NULL; - - switch (keyType) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: - comparFn = (order == TSDB_ORDER_ASC)? compareInt8Val:compareInt8ValDesc; - break; - case TSDB_DATA_TYPE_SMALLINT: - comparFn = (order == TSDB_ORDER_ASC)? compareInt16Val:compareInt16ValDesc; - break; - case TSDB_DATA_TYPE_INT: - comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - comparFn = (order == TSDB_ORDER_ASC)? compareInt64Val:compareInt64ValDesc; - break; - case TSDB_DATA_TYPE_FLOAT: - comparFn = (order == TSDB_ORDER_ASC)? compareFloatVal:compareFloatValDesc; - break; - case TSDB_DATA_TYPE_DOUBLE: - comparFn = (order == TSDB_ORDER_ASC)? compareDoubleVal:compareDoubleValDesc; - break; - case TSDB_DATA_TYPE_UTINYINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint8Val:compareUint8ValDesc; - break; - case TSDB_DATA_TYPE_USMALLINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint16Val:compareUint16ValDesc; - break; - case TSDB_DATA_TYPE_UINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint32Val:compareUint32ValDesc; - break; - case TSDB_DATA_TYPE_UBIGINT: - comparFn = (order == TSDB_ORDER_ASC)? compareUint64Val:compareUint64ValDesc; - break; - case TSDB_DATA_TYPE_BINARY: - comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedStr:compareLenPrefixedStrDesc; - break; - case TSDB_DATA_TYPE_NCHAR: - comparFn = (order == TSDB_ORDER_ASC)? compareLenPrefixedWStr:compareLenPrefixedWStrDesc; - break; - default: - comparFn = (order == TSDB_ORDER_ASC)? compareInt32Val:compareInt32ValDesc; - break; - } - - return comparFn; -} - -int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { - switch (type) { - case TSDB_DATA_TYPE_INT: DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); - case TSDB_DATA_TYPE_DOUBLE: DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); - case TSDB_DATA_TYPE_FLOAT: DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); - case TSDB_DATA_TYPE_BIGINT: DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); - case TSDB_DATA_TYPE_SMALLINT: DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); - case TSDB_DATA_TYPE_UTINYINT: DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); - case TSDB_DATA_TYPE_USMALLINT: DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); - case TSDB_DATA_TYPE_UINT: DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); - case TSDB_DATA_TYPE_UBIGINT: DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); - case TSDB_DATA_TYPE_NCHAR: { - tstr* t1 = (tstr*) f1; - tstr* t2 = (tstr*) f2; - - if (t1->len != t2->len) { - return t1->len > t2->len? 1:-1; - } - int32_t ret = memcmp((wchar_t*) t1, (wchar_t*) t2, t2->len); - if (ret == 0) { - return ret; - } - return (ret < 0) ? -1 : 1; - } - default: { // todo refactor - tstr* t1 = (tstr*) f1; - tstr* t2 = (tstr*) f2; - - if (t1->len != t2->len) { - return t1->len > t2->len? 1:-1; - } else { - int32_t ret = strncmp(t1->data, t2->data, t1->len); - if (ret == 0) { - return 0; - } else { - return ret < 0? -1:1; - } - } - } - } -} diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 41d911e4c7d068ca12997b3eb5d6f861b2bf2e08..f39b20b9347e7f1a441cd51d01c5297ebafef024 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -12,12 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#define _DEFAULT_SOURCE #include "tdataformat.h" -#include "ulog.h" -#include "talgo.h" #include "tcoding.h" -#include "wchar.h" -#include "tarray.h" +#include "tlog.h" static void dataColSetNEleNull(SDataCol *pCol, int nEle); #if 0 @@ -26,7 +25,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i #endif int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; - if(IS_VAR_DATA_TYPE(pCol->type)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { spaceNeeded += sizeof(VarDataOffsetT) * maxPoints; } #ifdef TD_SUPPORT_BITMAP @@ -37,11 +36,10 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += TYPE_BYTES[pCol->type]; #endif - if(pCol->spaceSize < spaceNeeded) { - void* ptr = realloc(pCol->pData, spaceNeeded); - if(ptr == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, - strerror(errno)); + if (pCol->spaceSize < spaceNeeded) { + void *ptr = realloc(pCol->pData, spaceNeeded); + if (ptr == NULL) { + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; } else { pCol->pData = ptr; @@ -67,8 +65,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { * Duplicate the schema and return a new object */ STSchema *tdDupSchema(const STSchema *pSchema) { - - int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); + int tlen = sizeof(STSchema) + sizeof(STColumn) * schemaNCols(pSchema); STSchema *tSchema = (STSchema *)malloc(tlen); if (tSchema == NULL) return NULL; @@ -99,8 +96,8 @@ int tdEncodeSchema(void **buf, STSchema *pSchema) { * Decode a schema from a binary. */ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { - int version = 0; - int numOfCols = 0; + int version = 0; + int numOfCols = 0; STSchemaBuilder schemaBuilder; buf = taosDecodeFixedI32(buf, &version); @@ -156,7 +153,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols >= pBuilder->tCols) { pBuilder->tCols *= 2; - STColumn* columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); + STColumn *columns = (STColumn *)realloc(pBuilder->columns, sizeof(STColumn) * pBuilder->tCols); if (columns == NULL) return -1; pBuilder->columns = columns; } @@ -167,7 +164,7 @@ int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int1 if (pBuilder->nCols == 0) { colSetOffset(pCol, 0); } else { - STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols-1]); + STColumn *pTCol = &(pBuilder->columns[pBuilder->nCols - 1]); colSetOffset(pCol, pTCol->offset + TYPE_BYTES[pTCol->type]); } @@ -259,7 +256,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->type = colType(pCol); pDataCol->colId = colColId(pCol); pDataCol->bytes = colBytes(pCol); - pDataCol->offset = colOffset(pCol) + 0; //TD_DATA_ROW_HEAD_SIZE; + pDataCol->offset = colOffset(pCol) + 0; // TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; } @@ -273,7 +270,7 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo return 0; } - if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1; + if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1; if (numOfRows > 0) { // Find the first not null value, fill all previouse values as NULL dataColSetNEleNull(pCol, numOfRows); @@ -304,7 +301,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) } bool isNEleNull(SDataCol *pCol, int nEle) { - if(isAllRowsNull(pCol)) return true; + if (isAllRowsNull(pCol)) return true; for (int i = 0; i < nEle; i++) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } @@ -371,7 +368,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { return NULL; } int i; - for(i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; pCols->cols[i].len = 0; pCols->cols[i].pData = NULL; @@ -387,10 +384,10 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + void *ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for(i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; i++) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; pCols->cols[i].spaceSize = 0; @@ -403,16 +400,16 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { for (i = 0; i < schemaNCols(pSchema); i++) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } - + return 0; } SDataCols *tdFreeDataCols(SDataCols *pCols) { int i; if (pCols) { - if(pCols->cols) { + if (pCols->cols) { int maxCols = pCols->maxCols; - for(i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; i++) { SDataCol *pCol = &pCols->cols[i]; tfree(pCol->pData); } @@ -440,7 +437,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { if (pDataCols->cols[i].len > 0) { - if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + if (tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { tdFreeDataCols(pRet); return NULL; } @@ -648,9 +645,9 @@ SKVRow tdKVRowDup(SKVRow row) { return trow; } -static int compareColIdx(const void* a, const void* b) { - const SColIdx* x = (const SColIdx*)a; - const SColIdx* y = (const SColIdx*)b; +static int compareColIdx(const void *a, const void *b) { + const SColIdx *x = (const SColIdx *)a; + const SColIdx *y = (const SColIdx *)b; if (x->colId > y->colId) { return 1; } @@ -660,15 +657,13 @@ static int compareColIdx(const void* a, const void* b) { return 0; } -void tdSortKVRowByColIdx(SKVRow row) { - qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); -} +void tdSortKVRowByColIdx(SKVRow row) { qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); } int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { SColIdx *pColIdx = NULL; SKVRow row = *orow; SKVRow nrow = NULL; - void * ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); + void *ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; @@ -700,7 +695,7 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { if (IS_VAR_DATA_TYPE(type)) { void *pOldVal = kvRowColVal(row, (SColIdx *)ptr); - if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place + if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place memcpy(pOldVal, value, varDataTLen(value)); } else { // need to reallocate the memory int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 970b6d954f1faa4f1f439f1f415e0d7059852311..e2880441be2585634a80c1c09d853a8830c73b35 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -1,10 +1,24 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "tep.h" -#include -#include "common.h" +#include "tcompare.h" #include "tglobal.h" -#include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); @@ -61,9 +75,8 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } -#define BitmapLen(_n) (((_n) + ((1<> NBIT) -int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { +int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { return pColumnInfoData->varmeta.length; @@ -121,14 +134,18 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; switch(type) { + case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} + case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} + case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} default: assert(0); } @@ -249,8 +266,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { } ASSERT(pColInfoData->nullbitmap == NULL); - pDataBlock->info.window.skey = *(TSKEY*) colDataGet(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*) colDataGet(pColInfoData, (pDataBlock->info.rows - 1)); + pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0); + pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); return 0; } @@ -262,8 +279,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); - uint32_t oldLen = colDataGetSize(pCol2, pDest->info.rows); - uint32_t newLen = colDataGetSize(pCol1, pSrc->info.rows); + uint32_t oldLen = colDataGetLength(pCol2, pDest->info.rows); + uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; char* tmp = realloc(pCol2->pData, newSize); @@ -287,7 +304,7 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - total += colDataGetSize(pColInfoData, pBlock->info.rows); + total += colDataGetLength(pColInfoData, pBlock->info.rows); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { total += sizeof(int32_t) * pBlock->info.rows; @@ -336,7 +353,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd if (isNull) { // do nothing } else { - char* p = colDataGet(pColInfoData, j); + char* p = colDataGetData(pColInfoData, j); size += varDataTLen(p); } @@ -401,7 +418,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); - char* p = colDataGet(pColData, j); + char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); } @@ -411,7 +428,6 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return pDst; } - /** * * +------------------+---------------+--------------------+ @@ -444,7 +460,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { pStart += BitmapLen(pBlock->info.rows); } - uint32_t dataSize = colDataGetSize(pCol, numOfRows); + uint32_t dataSize = colDataGetLength(pCol, numOfRows); *(int32_t*) pStart = dataSize; pStart += sizeof(int32_t); @@ -522,6 +538,22 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); } +SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { + SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + pSchema[i].bytes = pColInfoData->info.bytes; + pSchema[i].type = pColInfoData->info.type; + pSchema[i].colId = pColInfoData->info.colId; + } + + if (numOfCols != NULL) { + *numOfCols = pBlock->info.numOfCols; + } + + return pSchema; +} + double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; @@ -577,8 +609,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { } } - void* left1 = colDataGet(pColInfoData, left); - void* right1 = colDataGet(pColInfoData, right); + void* left1 = colDataGetData(pColInfoData, left); + void* right1 = colDataGetData(pColInfoData, right); switch(pColInfoData->info.type) { case TSDB_DATA_TYPE_INT: { @@ -617,7 +649,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return code; } } else { - char* p = colDataGet(pSrc, tupleIndex); + char* p = colDataGetData(pSrc, tupleIndex); code = colDataAppend(pDst, numOfRows, p, false); if (code != TSDB_CODE_SUCCESS) { return code; @@ -956,8 +988,8 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { // } // } -// void* left1 = colDataGet(pColInfoData, left); -// void* right1 = colDataGet(pColInfoData, right); +// void* left1 = colDataGetData(pColInfoData, left); +// void* right1 = colDataGetData(pColInfoData, right); // switch(pColInfoData->info.type) { // case TSDB_DATA_TYPE_INT: { @@ -1041,36 +1073,47 @@ void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { } } -int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(p->info.type)) { - char* tmp = realloc(p->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } +int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { + char* tmp = realloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - p->varmeta.offset = (int32_t*)tmp; - memset(p->varmeta.offset, 0, sizeof(int32_t) * numOfRows); + pColumn->varmeta.offset = (int32_t*)tmp; + memset(pColumn->varmeta.offset, 0, sizeof(int32_t) * numOfRows); - p->varmeta.length = 0; - p->varmeta.allocLen = 0; - tfree(p->pData); - } else { - char* tmp = realloc(p->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + pColumn->varmeta.length = 0; + pColumn->varmeta.allocLen = 0; + tfree(pColumn->pData); + } else { + char* tmp = realloc(pColumn->nullbitmap, BitmapLen(numOfRows)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - p->nullbitmap = tmp; - memset(p->nullbitmap, 0, BitmapLen(numOfRows)); + pColumn->nullbitmap = tmp; + memset(pColumn->nullbitmap, 0, BitmapLen(numOfRows)); - tmp = realloc(p->pData, numOfRows * p->info.bytes); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + tmp = realloc(pColumn->pData, numOfRows * pColumn->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumn->pData = tmp; + } + + return TSDB_CODE_SUCCESS; +} - p->pData = tmp; +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { + int32_t code = 0; + + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + code = blockDataEnsureColumnCapacity(p, numOfRows); + if (code) { + return code; } } @@ -1098,4 +1141,25 @@ void* blockDataDestroy(SSDataBlock* pBlock) { tfree(pBlock->pBlockAgg); tfree(pBlock); return NULL; -} \ No newline at end of file +} + +SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { + int32_t numOfCols = pDataBlock->info.numOfCols; + + SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + colInfo.info = p->info; + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + return pBlock; +} + +size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { + return pageSize / (blockDataGetSerialRowSize(pBlock) + blockDataGetSerialMetaSize(pBlock)); +} diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index afba1190babb7fcd1c2e48a1aa80568b7cfca213..db84f87b66c77fd636eb6df1d04efe6c82d108f8 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -14,19 +14,13 @@ */ #define _DEFAULT_SOURCE -#include "os.h" - -#include "taosdef.h" -#include "taoserror.h" +#include "tglobal.h" #include "tcompare.h" #include "tconfig.h" #include "tep.h" -#include "tglobal.h" -#include "tlocale.h" #include "tlog.h" -#include "ttimezone.h" -#include "tutil.h" -#include "ulog.h" + +SConfig *tsCfg = NULL; // cluster char tsFirst[TSDB_EP_LEN] = {0}; @@ -34,27 +28,24 @@ char tsSecond[TSDB_EP_LEN] = {0}; char tsLocalFqdn[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_EP_LEN] = {0}; // Local End Point, hostname:port uint16_t tsServerPort = 6030; +int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second -int8_t tsEnableTelemetryReporting = 0; -char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 128; +bool tsEnableTelemetryReporting = 0; // common int32_t tsRpcTimer = 300; int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default +bool tsRpcForceTcp = 1; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; int32_t tsNumOfCommitThreads = 4; float tsRatioOfQueryCores = 1.0f; -int8_t tsDaylight = 0; -int8_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; -int8_t tsEnableSlaveQuery = 1; -int8_t tsEnableAdjustMaster = 1; -int8_t tsPrintAuth = 0; +bool tsEnableSlaveQuery = 1; +bool tsPrintAuth = 0; + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. @@ -81,8 +72,6 @@ int32_t tsCompatibleModel = 1; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; -int8_t tsTscEnableRecordSql = 0; - // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp int32_t tsMaxNumOfOrderedResults = 100000; @@ -119,25 +108,20 @@ int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; // in retrieve blocking model, the retrieve threads will wait for the completion of the query processing. -int32_t tsRetrieveBlockingModel = 0; +bool tsRetrieveBlockingModel = 0; // last_row(*), first(*), last_row(ts, col1, col2) query, the result fields will be the original column name -int8_t tsKeepOriginalColumnName = 0; +bool tsKeepOriginalColumnName = 0; // long query death-lock -int8_t tsDeadLockKillQuery = 0; +bool tsDeadLockKillQuery = 0; // tsdb config // For backward compatibility bool tsdbForceKeepFile = false; -int32_t tsDiskCfgNum = 0; - -#ifndef _STORAGE -SDiskCfg tsDiskCfg[1]; -#else -SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; -#endif +int32_t tsDiskCfgNum = 0; +SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; /* * minimum scale for whole system, millisecond by default @@ -147,20 +131,7 @@ SDiskCfg tsDiskCfg[TFS_MAX_DISKS]; */ int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L}; -// system info -float tsTotalTmpDirGB = 0; -float tsTotalDataDirGB = 0; -float tsAvailTmpDirectorySpace = 0; -float tsAvailDataDirGB = 0; -float tsUsedDataDirGB = 0; -float tsReservedTmpDirectorySpace = 1.0f; -float tsMinimalDataDirGB = 2.0f; -int32_t tsTotalMemoryMB = 0; -uint32_t tsVersion = 0; - -// // lossy compress 6 -// char tsLossyColumns[32] = ""; // "float|double" means all float and double columns can be lossy compressed. set empty // can close lossy compress. // below option can take effect when tsLossyColumns not empty @@ -170,986 +141,396 @@ uint32_t tsMaxRange = 500; // max range uint32_t tsCurRange = 100; // range char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPRESSOR - -int32_t (*monStartSystemFp)() = NULL; -void (*monStopSystemFp)() = NULL; -void (*monExecuteSQLFp)(char *sql) = NULL; - -char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; - -static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; - -void taosSetAllDebugFlag() { - if (debugFlag != 0) { - mDebugFlag = debugFlag; - dDebugFlag = debugFlag; - vDebugFlag = debugFlag; - jniDebugFlag = debugFlag; - qDebugFlag = debugFlag; - rpcDebugFlag = debugFlag; - uDebugFlag = debugFlag; - sDebugFlag = debugFlag; - wDebugFlag = debugFlag; - tsdbDebugFlag = debugFlag; - cqDebugFlag = debugFlag; - uInfo("all debug flag are set to %d", debugFlag); - } +static void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { + tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); + tsDiskCfg[index].level = level; + tsDiskCfg[index].primary = primary; + uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); } -int32_t taosCfgDynamicOptions(char *msg) { - char *option, *value; - int32_t olen, vlen; - int32_t vint = 0; - - paGetToken(msg, &option, &olen); - if (olen == 0) return -1; +static void taosSetTfsCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); + if (pItem == NULL) return; - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) - vint = 135; - else { - vint = atoi(value); + int32_t size = taosArrayGetSize(pItem->array); + if (size <= 0) { + tsDiskCfgNum = 1; + taosAddDataDir(0, pItem->str, 0, 1); + } else { + tsDiskCfgNum = size < TFS_MAX_DISKS ? size : TFS_MAX_DISKS; + for (int32_t index = 0; index < tsDiskCfgNum; ++index) { + SDiskCfg *pCfg = taosArrayGet(pItem->array, index); + memcpy(&tsDiskCfg[index], pCfg, sizeof(SDiskCfg)); + } } +} - uInfo("change dynamic option: %s, value: %d", option, vint); +struct SConfig *taosGetCfg() { + return tsCfg; +} - for (int32_t i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - // if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (cfg->valType != TAOS_CFG_VTYPE_INT32 && cfg->valType != TAOS_CFG_VTYPE_INT8) continue; +static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { + char cfgDir[PATH_MAX] = {0}; + char cfgFile[PATH_MAX + 100] = {0}; - int32_t cfgLen = (int32_t)strlen(cfg->option); - if (cfgLen != olen) continue; - if (strncasecmp(option, cfg->option, olen) != 0) continue; - if (cfg->valType == TAOS_CFG_VTYPE_INT32) { - *((int32_t *)cfg->ptr) = vint; - } else { - *((int8_t *)cfg->ptr) = (int8_t)vint; - } + taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); + snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); - if (strncasecmp(cfg->option, "monitor", olen) == 0) { - if (1 == vint) { - if (monStartSystemFp) { - (*monStartSystemFp)(); - uInfo("monitor is enabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } else { - if (monStopSystemFp) { - (*monStopSystemFp)(); - uInfo("monitor is disabled"); - } else { - uError("monitor can't be updated, for monitor not initialized"); - } - } - return 0; - } - if (strncasecmp(cfg->option, "debugFlag", olen) == 0) { - taosSetAllDebugFlag(); + if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { + uError("failed to load from apollo url:%s since %s\n", apolloUrl, terrstr()); + return -1; + } + + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { + if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) { + uError("failed to load from config file:%s since %s\n", cfgFile, terrstr()); + return -1; } - return 0; } - if (strncasecmp(option, "resetlog", 8) == 0) { - taosResetLog(); - taosPrintCfg(); - return 0; + if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { + uError("failed to load from env file:%s since %s\n", envFile, terrstr()); + return -1; } - if (strncasecmp(option, "resetQueryCache", 15) == 0) { - if (monExecuteSQLFp) { - (*monExecuteSQLFp)("resetQueryCache"); - uInfo("resetquerycache is executed"); - } else { - uError("resetquerycache can't be executed, for monitor not started"); - } + if (cfgLoad(pCfg, CFG_STYPE_ENV_VAR, NULL) != 0) { + uError("failed to load from global env variables since %s\n", terrstr()); + return -1; } - return false; + return 0; } -void taosAddDataDir(int index, char *v1, int level, int primary) { - tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); - tsDiskCfg[index].level = level; - tsDiskCfg[index].primary = primary; - uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary); +static int32_t taosAddClientLogCfg(SConfig *pCfg) { + if (cfgAddDir(pCfg, "configDir", configDir, 1) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, 1) != 0) return -1; + if (cfgAddDir(pCfg, "logDir", tsLogDir, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, 1) != 0) return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, 1) != 0) return -1; + return 0; } -#ifndef _STORAGE -void taosReadDataDirCfg(char *v1, char *v2, char *v3) { - if (tsDiskCfgNum == 1) { - SDiskCfg *cfg = &tsDiskCfg[0]; - uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1); - } - taosAddDataDir(0, v1, 0, 1); - tsDiskCfgNum = 1; +static int32_t taosAddServerLogCfg(SConfig *pCfg) { + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, 0) != 0) return -1; + return 0; } -void taosPrintDataDirCfg() { - for (int i = 0; i < tsDiskCfgNum; ++i) { - SDiskCfg *cfg = &tsDiskCfg[i]; - uInfo(" dataDir: %s", cfg->dir); - } +static int32_t taosAddClientCfg(SConfig *pCfg) { + char defaultFqdn[TSDB_FQDN_LEN] = {0}; + int32_t defaultServerPort = 6030; + char defaultFirstEp[TSDB_EP_LEN] = {0}; + char defaultSecondEp[TSDB_EP_LEN] = {0}; + + if (taosGetFqdn(defaultFqdn) != 0) return -1; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + snprintf(defaultSecondEp, TSDB_EP_LEN, "%s:%d", defaultFqdn, defaultServerPort); + + if (cfgAddString(pCfg, "firstEp", defaultFirstEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", defaultSecondEp, 1) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfThreadsPerCore", tsNumOfThreadsPerCore, 0, 10, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1; + if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0) + return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; + return 0; } -#endif -static void taosCheckDataDirCfg() { - if (tsDiskCfgNum <= 0) { - taosAddDataDir(0, tsDataDir, 0, 1); - tsDiskCfgNum = 1; - uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir); - } +static int32_t taosAddSystemCfg(SConfig *pCfg) { + SysNameInfo info = taosGetSysNameInfo(); + + if (cfgAddTimezone(pCfg, "timezone", tsTimezone) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 0, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCores", tsNumOfCores, 1, 100000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSize, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "totalMemory(MB)", tsTotalMemoryMB, 0, INT32_MAX, 1) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, 1) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, 1) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; + + if (cfgAddString(pCfg, "version", version, 1) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, 1) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, 1) != 0) return -1; + if (cfgAddString(pCfg, "gitinfoOfInternal", gitinfoOfInternal, 1) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, 1) != 0) return -1; + return 0; } -static void doInitGlobalConfig(void) { - osInit(); - srand(taosSafeRand()); - - SGlobalCfg cfg = {0}; - - // ip address - cfg.option = "firstEp"; - cfg.ptr = tsFirst; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "secondEp"; - cfg.ptr = tsSecond; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_EP_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fqdn"; - cfg.ptr = tsLocalFqdn; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FQDN_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // port - cfg.option = "serverPort"; - cfg.ptr = &tsServerPort; - cfg.valType = TAOS_CFG_VTYPE_UINT16; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65056; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "supportVnodes"; - cfg.ptr = &tsNumOfSupportVnodes; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // directory - cfg.option = "configDir"; - cfg.ptr = configDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logDir"; - cfg.ptr = tsLogDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "scriptDir"; - cfg.ptr = tsScriptDir; - cfg.valType = TAOS_CFG_VTYPE_DIRECTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dataDir"; - cfg.ptr = tsDataDir; - cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_FILENAME_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // dnode configs - cfg.option = "numOfThreadsPerCore"; - cfg.ptr = &tsNumOfThreadsPerCore; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "numOfCommitThreads"; - cfg.ptr = &tsNumOfCommitThreads; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 1; - cfg.maxValue = 100; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "ratioOfQueryCores"; - cfg.ptr = &tsRatioOfQueryCores; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0.0f; - cfg.maxValue = 2.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxNumOfDistinctRes"; - cfg.ptr = &tsMaxNumOfDistinctResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 10 * 10000; - cfg.maxValue = 10000 * 10000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "telemetryReporting"; - cfg.ptr = &tsEnableTelemetryReporting; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // timer - cfg.option = "maxTmrCtrl"; - cfg.ptr = &tsMaxTmrCtrl; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 8; - cfg.maxValue = 2048; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcTimer"; - cfg.ptr = &tsRpcTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 3000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "rpcForceTcp"; - cfg.ptr = &tsRpcForceTcp; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcMaxTime"; - cfg.ptr = &tsRpcMaxTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 100; - cfg.maxValue = 7200; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "statusInterval"; - cfg.ptr = &tsStatusInterval; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 10; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "shellActivityTimer"; - cfg.ptr = &tsShellActivityTimer; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 120; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_SECOND; - taosAddConfigOption(cfg); - - cfg.option = "minSlidingTime"; - cfg.ptr = &tsMinSlidingTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "minIntervalTime"; - cfg.ptr = &tsMinIntervalTime; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 1000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxStreamCompDelay"; - cfg.ptr = &tsMaxStreamComputDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "maxFirstStreamCompDelay"; - cfg.ptr = &tsStreamCompStartDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1000; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - taosAddConfigOption(cfg); - - cfg.option = "retryStreamCompDelay"; - cfg.ptr = &tsRetryStreamCompDelay; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 1000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_MS; - - taosAddConfigOption(cfg); - cfg.option = "streamCompDelayRatio"; - cfg.ptr = &tsStreamComputDelayRatio; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.1f; - cfg.maxValue = 0.9f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressMsgSize"; - cfg.ptr = &tsCompressMsgSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "compressColData"; - cfg.ptr = &tsCompressColData; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 100000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxWildCardsLength"; - cfg.ptr = &tsMaxWildCardsLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxRegexStringLen"; - cfg.ptr = &tsMaxRegexStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = TSDB_MAX_FIELD_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "maxNumOfOrderedRes"; - cfg.ptr = &tsMaxNumOfOrderedResults; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "queryBufferSize"; - cfg.ptr = &tsQueryBufferSize; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = -1; - cfg.maxValue = 500000000000.0f; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - - cfg.option = "retrieveBlockingModel"; - cfg.ptr = &tsRetrieveBlockingModel; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "keepColumnName"; - cfg.ptr = &tsKeepOriginalColumnName; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // locale & charset - cfg.option = "timezone"; - cfg.ptr = tsTimezone; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_TIMEZONE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "locale"; - cfg.ptr = tsLocale; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "charset"; - cfg.ptr = tsCharset; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = TSDB_LOCALE_LEN; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // connect configs - cfg.option = "maxShellConns"; - cfg.ptr = &tsMaxShellConns; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 10; - cfg.maxValue = 50000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxConnections"; - cfg.ptr = &tsMaxConnections; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 1; - cfg.maxValue = 100000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "minimalLogDirGB"; - cfg.ptr = &tsMinimalLogDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalTmpDirGB"; - cfg.ptr = &tsReservedTmpDirectorySpace; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "minimalDataDirGB"; - cfg.ptr = &tsMinimalDataDirGB; - cfg.valType = TAOS_CFG_VTYPE_FLOAT; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0.001f; - cfg.maxValue = 10000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_GB; - taosAddConfigOption(cfg); - - cfg.option = "slaveQuery"; - cfg.ptr = &tsEnableSlaveQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // debug flag - cfg.option = "numOfLogLines"; - cfg.ptr = &tsNumOfLogLines; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1000; - cfg.maxValue = 2000000000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "logKeepDays"; - cfg.ptr = &tsLogKeepDays; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = -365000; - cfg.maxValue = 365000; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "asyncLog"; - cfg.ptr = &tsAsyncLog; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "debugFlag"; - cfg.ptr = &debugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "mDebugFlag"; - cfg.ptr = &mDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "dDebugFlag"; - cfg.ptr = &dDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "sDebugFlag"; - cfg.ptr = &sDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "wDebugFlag"; - cfg.ptr = &wDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "rpcDebugFlag"; - cfg.ptr = &rpcDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tmrDebugFlag"; - cfg.ptr = &tmrDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cDebugFlag"; - cfg.ptr = &cDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "jniDebugFlag"; - cfg.ptr = &jniDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "uDebugFlag"; - cfg.ptr = &uDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "qDebugFlag"; - cfg.ptr = &qDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "vDebugFlag"; - cfg.ptr = &vDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tsdbDebugFlag"; - cfg.ptr = &tsdbDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "cqDebugFlag"; - cfg.ptr = &cqDebugFlag; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; - cfg.minValue = 0; - cfg.maxValue = 255; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableRecordSql"; - cfg.ptr = &tsTscEnableRecordSql; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "enableCoreFile"; - cfg.ptr = &tsEnableCoreFile; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // version info - cfg.option = "gitinfo"; - cfg.ptr = gitinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "gitinfoOfInternal"; - cfg.ptr = gitinfoOfInternal; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "buildinfo"; - cfg.ptr = buildinfo; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "version"; - cfg.ptr = version; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxBinaryDisplayWidth"; - cfg.ptr = &tsMaxBinaryDisplayWidth; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 1; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "tempDir"; - cfg.ptr = tsTempDir; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = PATH_MAX; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - // enable kill long query - cfg.option = "deadLockKillQuery"; - cfg.ptr = &tsDeadLockKillQuery; - cfg.valType = TAOS_CFG_VTYPE_INT8; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = 0; - cfg.maxValue = 1; - cfg.ptrLength = 1; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - -#ifdef TD_TSZ - // lossy compress - cfg.option = "lossyColumns"; - cfg.ptr = lossyColumns; - cfg.valType = TAOS_CFG_VTYPE_STRING; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 0; - cfg.ptrLength = tListLen(lossyColumns); - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "fPrecision"; - cfg.ptr = &fPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - - taosAddConfigOption(cfg); - - cfg.option = "dPrecision"; - cfg.ptr = &dPrecision; - cfg.valType = TAOS_CFG_VTYPE_DOUBLE; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = MIN_FLOAT; - cfg.maxValue = MAX_FLOAT; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "maxRange"; - cfg.ptr = &maxRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - - cfg.option = "range"; - cfg.ptr = &curRange; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; - cfg.minValue = 0; - cfg.maxValue = 65536; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_NONE; - taosAddConfigOption(cfg); - assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM); -#else - // assert(tsGlobalConfigNum == TSDB_CFG_MAX_NUM - 5); -#endif +static int32_t taosAddServerCfg(SConfig *pCfg) { + if (cfgAddInt32(pCfg, "supportVnodes", 256, 0, 65536, 0) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 100, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "ratioOfQueryCores", tsRatioOfQueryCores, 0, 2, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelemetryReporting, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxConnections", tsMaxConnections, 1, 100000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0) != 0) return -1; + if (cfgAddFloat(pCfg, "streamCompDelayRatio", tsStreamComputDelayRatio, 0.1, 0.9, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, 0) != 0) return -1; + if (cfgAddBool(pCfg, "retrieveBlockingModel", tsRetrieveBlockingModel, 0) != 0) return -1; + if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, 0) != 0) return -1; + if (cfgAddBool(pCfg, "slaveQuery", tsEnableSlaveQuery, 0) != 0) return -1; + if (cfgAddBool(pCfg, "deadLockKillQuery", tsDeadLockKillQuery, 0) != 0) return -1; + return 0; } -void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } +static void taosSetClientLogCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "logDir"); + tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + tsLogSpace.reserved = cfgGetItem(pCfg, "minimalLogDirGB")->fval; + tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; + tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; + tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; + uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; + rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; + jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; +} + +static void taosSetServerLogCfg(SConfig *pCfg) { + dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; + vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; + mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; + wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; + tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; + tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; + fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; +} -int32_t taosCheckAndPrintCfg() { - SEp ep = {0}; - if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - taosSetAllDebugFlag(); +static void taosSetClientCfg(SConfig *pCfg) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_EP_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + + SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); + SEp secondEp = {0}; + taosGetFqdnPortFromEp(pSecondpItem->str, &secondEp); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); + cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); + + tstrncpy(tsLogDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalTempDirGB")->fval; + + tsNumOfThreadsPerCore = cfgGetItem(pCfg, "maxTmrCtrl")->fval; + tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; + tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; + tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; + tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32; + tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->bval; + tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; + tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; + tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32; + tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32; + tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; + tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; +} + +static void taosSetSystemCfg(SConfig *pCfg) { + SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); + osSetTimezone(pItem->str); + uDebug("timezone format changed from %s to %s", pItem->str, tsTimezone); + cfgSetItem(pCfg, "timezone", tsTimezone, pItem->stype); + + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + + if (tsNumOfCores <= 1) { + tsNumOfCores = 2; } - if (tsLocalFqdn[0] == 0) { - taosGetFqdn(tsLocalFqdn); + bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; + taosSetConsoleEcho(enableCore); + + // todo + tsVersion = 30000000; +} + +static void taosSetServerCfg(SConfig *pCfg) { + tstrncpy(tsDataDir, cfgGetItem(pCfg, "dataDir")->str, PATH_MAX); + taosExpandDir(tsDataDir, tsDataDir, PATH_MAX); + + tsTempSpace.reserved = cfgGetItem(pCfg, "minimalDataDirGB")->fval; + tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + tsRatioOfQueryCores = cfgGetItem(pCfg, "ratioOfQueryCores")->fval; + tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + tsEnableTelemetryReporting = cfgGetItem(pCfg, "telemetryReporting")->bval; + tsMaxConnections = cfgGetItem(pCfg, "maxConnections")->i32; + tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; + tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; + tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; + tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; + tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; + tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; + tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; + tsStreamComputDelayRatio = cfgGetItem(pCfg, "streamCompDelayRatio")->fval; + tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; + tsRetrieveBlockingModel = cfgGetItem(pCfg, "retrieveBlockingModel")->bval; + tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + tsEnableSlaveQuery = cfgGetItem(pCfg, "slaveQuery")->bval; + tsDeadLockKillQuery = cfgGetItem(pCfg, "deadLockKillQuery")->bval; + + if (tsQueryBufferSize >= 0) { + tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } +} - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - uInfo("localEp is: %s", tsLocalEp); +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, + const char *apolloUrl, SArray *pArgs, bool tsc) { + osInit(); + + SConfig *pCfg = cfgInit(); + if (pCfg == NULL) return -1; - if (tsFirst[0] == 0) { - strcpy(tsFirst, tsLocalEp); + if (tsc) { + if (taosAddClientLogCfg(pCfg) != 0) return -1; } else { - taosGetFqdnPortFromEp(tsFirst, &ep); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); + if (taosAddClientLogCfg(pCfg) != 0) return -1; + if (taosAddServerLogCfg(pCfg) != 0) return -1; + } + + if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(pCfg); + return -1; } - if (tsSecond[0] == 0) { - strcpy(tsSecond, tsLocalEp); + if (cfgLoadArray(pCfg, pArgs) != 0) { + uError("failed to load cfg from array since %s", terrstr()); + cfgCleanup(pCfg); + return -1; + } + + if (tsc) { + taosSetClientLogCfg(pCfg); } else { - taosGetFqdnPortFromEp(tsSecond, &ep); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); + taosSetClientLogCfg(pCfg); + taosSetServerLogCfg(pCfg); } - taosCheckDataDirCfg(); + taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32); - if (taosDirExist(tsTempDir) != 0) { + if (taosInitLog(logname, logFileNum) != 0) { + printf("failed to init log file since %s\n", terrstr()); + cfgCleanup(pCfg); return -1; } - taosGetSystemInfo(); + cfgCleanup(pCfg); + return 0; +} - tsSetLocale(); +int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { + if (tsCfg != NULL) return 0; + tsCfg = cfgInit(); - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus == TAOS_CFG_CSTATUS_FILE) { - tsSetTimeZone(); + if (tsc) { + if (taosAddClientLogCfg(tsCfg) != 0) return -1; + if (taosAddClientCfg(tsCfg) != 0) return -1; + } else { + if (taosAddClientCfg(tsCfg) != 0) return -1; + if (taosAddServerCfg(tsCfg) != 0) return -1; + if (taosAddClientLogCfg(tsCfg) != 0) return -1; + if (taosAddServerLogCfg(tsCfg) != 0) return -1; } + taosAddSystemCfg(tsCfg); - if (tsNumOfCores <= 0) { - tsNumOfCores = 1; + if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) { + uError("failed to load cfg since %s", terrstr()); + cfgCleanup(tsCfg); + tsCfg = NULL; + return -1; } - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; + if (cfgLoadArray(tsCfg, pArgs) != 0) { + uError("failed to load cfg from array since %s", terrstr()); + cfgCleanup(tsCfg); + return -1; } - uInfo(" check global cfg completed"); - uInfo("=================================="); - taosPrintCfg(); + if (tsc) { + taosSetClientCfg(tsCfg); + } else { + taosSetClientCfg(tsCfg); + taosSetServerCfg(tsCfg); + taosSetTfsCfg(tsCfg); + } + taosSetSystemCfg(tsCfg); + cfgDumpCfg(tsCfg, tsc, false); return 0; } -/* - * alter dnode 1 balance "vnode:1-dnode:2" - */ - -bool taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId) { - int len = (int)strlen(option); - if (strncasecmp(option, "vnode:", 6) != 0) { - return false; - } - - int pos = 0; - for (; pos < len; ++pos) { - if (option[pos] == '-') break; +void taosCleanupCfg() { + if (tsCfg) { + cfgCleanup(tsCfg); + tsCfg = NULL; } +} - if (++pos >= len) return false; - if (strncasecmp(option + pos, "dnode:", 6) != 0) { - return false; +void taosCfgDynamicOptions(const char *option, const char *value) { + if (strcasecmp(option, "debugFlag") == 0) { + int32_t debugFlag = atoi(value); + taosSetAllDebugFlag(debugFlag); } - *vnodeId = strtol(option + 6, NULL, 10); - *dnodeId = strtol(option + pos + 6, NULL, 10); - if (*vnodeId <= 1 || *dnodeId <= 0) { - return false; + if (strcasecmp(option, "resetlog") == 0) { + taosResetLog(); + cfgDumpCfg(tsCfg, 1, false); } - - return true; -} +} \ No newline at end of file diff --git a/source/common/src/tlocale.c b/source/common/src/tlocale.c deleted file mode 100644 index a98a46b28a658924042b35df93d17db18b2c2f54..0000000000000000000000000000000000000000 --- a/source/common/src/tlocale.c +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -/** - * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of - * both the TDengine Server and the Client may be interrupted. - * - * In case that the setLocale failed to be executed, the right charset needs to be set. - */ -void tsSetLocale() { - char *locale = setlocale(LC_CTYPE, tsLocale); - - // default locale or user specified locale is not valid, abort launch - if (locale == NULL) { - uError("Invalid locale:%s, please set the valid locale in config file", tsLocale); - } - - if (strlen(tsCharset) == 0) { - uError("failed to get charset, please set the valid charset in config file"); - exit(-1); - } - - if (!taosValidateEncodec(tsCharset)) { - uError("Invalid charset:%s, please set the valid charset in config file", tsCharset); - exit(-1); - } -} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8101c18ebf43ce78fb119c77d8eeca6aea6fbd6e..8a3cd0a7183b935da6d81f6f6651c954942f4a3f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tmsg.h" #undef TD_MSG_NUMBER_ @@ -1523,6 +1524,10 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + if (pRsp->vgNum <= 0) { + return 0; + } + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); if (pRsp->pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/common/src/tmsgtype.c b/source/common/src/tmsgtype.c index 6e845e7e9e4f1a7c4b0474e51e11c6e6e3d8f6c9..3ca0f00fc20df54226eb026e2c92e19144de47b2 100644 --- a/source/common/src/tmsgtype.c +++ b/source/common/src/tmsgtype.c @@ -13,6 +13,6 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #define TSDB_SQL_C - #include "tmsgtype.h" diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f6892b26bdb0fc167548d1d95287e5f21bbf6678..e061862856702c8015470afb496bd5e6734e49b6 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -1,9 +1,20 @@ -#include -#include "os.h" -#include "tutil.h" - +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "tname.h" -#include "tmsg.h" #define VALID_NAME_TYPE(x) ((x) == TSDB_DB_NAME_T || (x) == TSDB_TABLE_NAME_T) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 91d91f0714d475d427dd7087b27d64fb3f60faf6..861b4dc09334beb2d62e7e0677519c448855a38f 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "trow.h" -#include "tarray.h" const uint8_t tdVTypeByte[3] = { TD_VTYPE_NORM_BYTE, // TD_VTYPE_NORM diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index f9755a43b9c4a53473637110bd683556d6c64b10..460c4a6fc08ea07b77a9feec970a87be501e7fd7 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -13,20 +13,15 @@ * along with this program. If not, see . */ -#define _BSD_SOURCE - #ifdef DARWIN #define _XOPEN_SOURCE #else #define _XOPEN_SOURCE 500 #endif +#define _BSD_SOURCE #define _DEFAULT_SOURCE - -#include "os.h" -#include "taosdef.h" #include "ttime.h" -#include "tutil.h" /* * mktime64 - Converts date to seconds. @@ -48,50 +43,46 @@ * An encoding of midnight at the end of the day as 24:00:00 - ie. midnight * tomorrow - (allowable under ISO 8601) is supported. */ -static int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, - const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec, int64_t time_zone) -{ - unsigned int mon = mon0, year = year0; +static int64_t user_mktime64(const uint32_t year0, const uint32_t mon0, const uint32_t day, const uint32_t hour, + const uint32_t min, const uint32_t sec, int64_t time_zone) { + uint32_t mon = mon0, year = year0; /* 1..12 -> 11,12,1..10 */ - if (0 >= (int) (mon -= 2)) { - mon += 12; /* Puts Feb last since it has leap day */ + if (0 >= (int32_t)(mon -= 2)) { + mon += 12; /* Puts Feb last since it has leap day */ year -= 1; } - //int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + - // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); + // int64_t res = (((((int64_t) (year/4 - year/100 + year/400 + 367*mon/12 + day) + + // year*365 - 719499)*24 + hour)*60 + min)*60 + sec); int64_t res; - res = 367*((int64_t)mon)/12; - res += year/4 - year/100 + year/400 + day + ((int64_t)year)*365 - 719499; - res = res*24; - res = ((res + hour) * 60 + min) * 60 + sec; + res = 367 * ((int64_t)mon) / 12; + res += year / 4 - year / 100 + year / 400 + day + ((int64_t)year) * 365 - 719499; + res = res * 24; + res = ((res + hour) * 60 + min) * 60 + sec; return (res + time_zone); } // ==== mktime() kernel code =================// static int64_t m_deltaUtc = 0; -void deltaToUtcInitOnce() { +void deltaToUtcInitOnce() { struct tm tm = {0}; - (void)strptime("1970-01-01 00:00:00", (const char *)("%Y-%m-%d %H:%M:%S"), &tm); + (void)strptime("1970-01-01 00:00:00", (const char*)("%Y-%m-%d %H:%M:%S"), &tm); m_deltaUtc = (int64_t)mktime(&tm); - //printf("====delta:%lld\n\n", seconds); + // printf("====delta:%lld\n\n", seconds); } static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); static int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec); static int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec); -static char* forwardToTimeStringEnd(char* str); -static bool checkTzPresent(const char *str, int32_t len); +static char* forwardToTimeStringEnd(char* str); +static bool checkTzPresent(const char* str, int32_t len); -static int32_t (*parseLocaltimeFp[]) (char* timestr, int64_t* time, int32_t timePrec) = { - parseLocaltime, - parseLocaltimeDst -}; +static int32_t (*parseLocaltimeFp[])(char* timestr, int64_t* time, int32_t timePrec) = {parseLocaltime, + parseLocaltimeDst}; int32_t taosParseTime(const char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ @@ -108,9 +99,9 @@ bool checkTzPresent(const char* str, int32_t len) { char* seg = forwardToTimeStringEnd((char*)str); int32_t seg_len = len - (int32_t)(seg - str); - char *c = &seg[seg_len - 1]; - for (int i = 0; i < seg_len; ++i) { - if (*c == 'Z' || *c == 'z' || *c == '+' || *c == '-') { + char* c = &seg[seg_len - 1]; + for (int32_t i = 0; i < seg_len; ++i) { + if (*c == 'Z' || *c == 'z' || *c == '+' || *c == '-') { return true; } c--; @@ -204,13 +195,12 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { i += 2; } - //return error if there're illegal charaters after min(2 Digits) - char *minStr = &str[i]; + // return error if there're illegal charaters after min(2 Digits) + char* minStr = &str[i]; if (minStr[1] != '\0' && minStr[2] != '\0') { - return -1; + return -1; } - int64_t minute = strnatoi(&str[i], 2); if (minute > 59) { return -1; @@ -238,9 +228,8 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { * 2013-04-12T15:52:01.123+0800 */ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim) { - - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); int64_t tzOffset = 0; struct tm tm = {0}; @@ -260,8 +249,8 @@ int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, ch /* mktime will be affected by TZ, set by using taos_options */ #ifdef WINDOWS - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); - //int64_t seconds = gmtime(&tm); + int64_t seconds = user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0); + // int64_t seconds = gmtime(&tm); #else int64_t seconds = timegm(&tm); #endif @@ -325,8 +314,9 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { #endif #endif - int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); - + int64_t seconds = + user_mktime64(tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone); + int64_t fraction = 0; if (*str == '.') { @@ -336,8 +326,8 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) { } } - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); *time = factor * seconds + fraction; return 0; @@ -355,7 +345,7 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { /* mktime will be affected by TZ, set by using taos_options */ int64_t seconds = mktime(&tm); - + int64_t fraction = 0; if (*str == '.') { @@ -365,27 +355,22 @@ int32_t parseLocaltimeDst(char* timestr, int64_t* time, int32_t timePrec) { } } - int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : - (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); + int64_t factor = + (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); *time = factor * seconds + fraction; return 0; } int64_t convertTimePrecision(int64_t time, int32_t fromPrecision, int32_t toPrecision) { - assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || - fromPrecision == TSDB_TIME_PRECISION_MICRO || + assert(fromPrecision == TSDB_TIME_PRECISION_MILLI || fromPrecision == TSDB_TIME_PRECISION_MICRO || fromPrecision == TSDB_TIME_PRECISION_NANO); - assert(toPrecision == TSDB_TIME_PRECISION_MILLI || - toPrecision == TSDB_TIME_PRECISION_MICRO || + assert(toPrecision == TSDB_TIME_PRECISION_MILLI || toPrecision == TSDB_TIME_PRECISION_MICRO || toPrecision == TSDB_TIME_PRECISION_NANO); - static double factors[3][3] = { {1., 1000., 1000000.}, - {1.0 / 1000, 1., 1000.}, - {1.0 / 1000000, 1.0 / 1000, 1.} }; + static double factors[3][3] = {{1., 1000., 1000000.}, {1.0 / 1000, 1., 1000.}, {1.0 / 1000000, 1.0 / 1000, 1.}}; return (int64_t)((double)time * factors[fromPrecision][toPrecision]); } static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { - switch (unit) { case 's': (*result) = convertTimePrecision(val * MILLISECOND_PER_SECOND, TSDB_TIME_PRECISION_MILLI, timePrecision); @@ -432,7 +417,8 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time * d - Days (24 hours) * w - Weeks (7 days) */ -int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) { +int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* duration, char* unit, + int32_t timePrecision) { errno = 0; char* endPtr = NULL; @@ -479,9 +465,9 @@ int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) { } struct tm tm; - time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); + time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision)); localtime_r(&tt, &tm); - int mon = tm.tm_year * 12 + tm.tm_mon + (int)duration; + int32_t mon = tm.tm_year * 12 + tm.tm_mon + (int32_t)duration; tm.tm_year = mon / 12; tm.tm_mon = mon % 12; @@ -502,13 +488,13 @@ int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; - time_t t = (time_t)skey; + time_t t = (time_t)skey; localtime_r(&t, &tm); - int smon = tm.tm_year * 12 + tm.tm_mon; + int32_t smon = tm.tm_year * 12 + tm.tm_mon; t = (time_t)ekey; localtime_r(&t, &tm); - int emon = tm.tm_year * 12 + tm.tm_mon; + int32_t emon = tm.tm_year * 12 + tm.tm_mon; if (unit == 'y') { interval *= 12; @@ -527,7 +513,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') { start /= (int64_t)(TSDB_TICK_PER_SECOND(precision)); struct tm tm; - time_t tt = (time_t)start; + time_t tt = (time_t)start; localtime_r(&tt, &tm); tm.tm_sec = 0; tm.tm_min = 0; @@ -536,10 +522,10 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (pInterval->slidingUnit == 'y') { tm.tm_mon = 0; - tm.tm_year = (int)(tm.tm_year / pInterval->sliding * pInterval->sliding); + tm.tm_year = (int32_t)(tm.tm_year / pInterval->sliding * pInterval->sliding); } else { - int mon = tm.tm_year * 12 + tm.tm_mon; - mon = (int)(mon / pInterval->sliding * pInterval->sliding); + int32_t mon = tm.tm_year * 12 + tm.tm_mon; + mon = (int32_t)(mon / pInterval->sliding * pInterval->sliding); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; } @@ -552,17 +538,17 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio start = (delta / pInterval->sliding + factor) * pInterval->sliding; if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') { - /* - * here we revised the start time of day according to the local time zone, - * but in case of DST, the start time of one day need to be dynamically decided. - */ + /* + * here we revised the start time of day according to the local time zone, + * but in case of DST, the start time of one day need to be dynamically decided. + */ // todo refactor to extract function that is available for Linux/Windows/Mac platform - #if defined(WINDOWS) && _MSC_VER >= 1900 +#if defined(WINDOWS) && _MSC_VER >= 1900 // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 int64_t timezone = _timezone; int32_t daylight = _daylight; char** tzname = _tzname; - #endif +#endif start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision)); } @@ -573,7 +559,7 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { end = start + pInterval->interval - 1; - while(end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window + while (end < t && ((start + pInterval->sliding) <= INT64_MAX)) { // move forward to the correct time window start += pInterval->sliding; if (start < 0 || INT64_MAX - start > pInterval->interval - 1) { @@ -606,8 +592,8 @@ int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precisio // and the parameter can also be a variable. const char* fmtts(int64_t ts) { static char buf[96]; - size_t pos = 0; - struct tm tm; + size_t pos = 0; + struct tm tm; if (ts > -62135625943 && ts < 32503651200) { time_t t = (time_t)ts; @@ -624,7 +610,7 @@ const char* fmtts(int64_t ts) { buf[pos++] = ' '; } pos += strftime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm); - pos += sprintf(buf + pos, ".%03d", (int)(ts % 1000)); + pos += sprintf(buf + pos, ".%03d", (int32_t)(ts % 1000)); } { @@ -636,7 +622,7 @@ const char* fmtts(int64_t ts) { buf[pos++] = ' '; } pos += strftime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm); - pos += sprintf(buf + pos, ".%06d", (int)(ts % 1000000)); + pos += sprintf(buf + pos, ".%06d", (int32_t)(ts % 1000000)); } return buf; diff --git a/source/common/src/ttimezone.c b/source/common/src/ttimezone.c deleted file mode 100644 index c45e39c20d458bff58d35c8e76848051617da8eb..0000000000000000000000000000000000000000 --- a/source/common/src/ttimezone.c +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tglobal.h" -#include "tconfig.h" -#include "tutil.h" - -// TODO refactor to set the tz value through parameter -void tsSetTimeZone() { - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone != NULL) { - uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); - } - -#ifdef WINDOWS - char winStr[TSDB_LOCALE_LEN * 2]; - sprintf(winStr, "TZ=%s", tsTimezone); - putenv(winStr); -#else - setenv("TZ", tsTimezone, 1); -#endif - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ -#ifdef _MSC_VER -#if _MSC_VER >= 1900 - // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 - int64_t timezone = _timezone; - int32_t daylight = _daylight; - char **tzname = _tzname; -#endif -#endif - - int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); - tz += daylight; - - /* - * format: - * (CST, +0800) - * (BST, +0100) - */ - sprintf(tsTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - tsDaylight = daylight; - - uInfo("timezone format changed to %s", tsTimezone); -} diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 6d57992c35bab4b8cea04888f1cf78ab9417bcb2..f20e91168162757f98529de72f5e293843df0c3c 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -1,11 +1,25 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE #include "ttszip.h" -#include #include "taoserror.h" #include "tcompression.h" -#include "tutil.h" static int32_t getDataStartOffset(); -static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); +static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); @@ -22,10 +36,11 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { } pTSBuf->autoDelete = autoDelete; - + taosGetTmpfilePath(tsTempDir, "join", pTSBuf->path); - pTSBuf->f = fopen(pTSBuf->path, "wb+"); - if (pTSBuf->f == NULL) { + // pTSBuf->pFile = fopen(pTSBuf->path, "wb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } @@ -33,20 +48,20 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { if (!autoDelete) { remove(pTSBuf->path); } - + if (NULL == allocResForTSBuf(pTSBuf)) { return NULL; } - + // update the header info STSBufFileHeader header = {.magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = TSDB_ORDER_ASC}; STSBufUpdateHeader(pTSBuf, &header); - + tsBufResetPos(pTSBuf); pTSBuf->cur.order = TSDB_ORDER_ASC; pTSBuf->tsOrder = order; - + return pTSBuf; } @@ -57,24 +72,25 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { } pTSBuf->autoDelete = autoDelete; - + tstrncpy(pTSBuf->path, path, sizeof(pTSBuf->path)); - - pTSBuf->f = fopen(pTSBuf->path, "rb+"); - if (pTSBuf->f == NULL) { + + // pTSBuf->pFile = fopen(pTSBuf->path, "rb+"); + pTSBuf->pFile = taosOpenFile(pTSBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pTSBuf->pFile == NULL) { free(pTSBuf); return NULL; } - + if (allocResForTSBuf(pTSBuf) == NULL) { return NULL; } - + // validate the file magic number STSBufFileHeader header = {0}; - int32_t ret = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); UNUSED(ret); - size_t sz = fread(&header, 1, sizeof(STSBufFileHeader), pTSBuf->f); + size_t sz = taosReadFile(pTSBuf->pFile, &header, sizeof(STSBufFileHeader)); UNUSED(sz); // invalid file @@ -82,7 +98,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufDestroy(pTSBuf); return NULL; } - + if (header.numOfGroup > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfGroup; STSGroupBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * pTSBuf->numOfAlloc); @@ -90,57 +106,58 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->pData = tmp; } - + pTSBuf->numOfGroups = header.numOfGroup; - + // check the ts order pTSBuf->tsOrder = header.tsOrder; if (pTSBuf->tsOrder != TSDB_ORDER_ASC && pTSBuf->tsOrder != TSDB_ORDER_DESC) { -// tscError("invalid order info in buf:%d", pTSBuf->tsOrder); + // tscError("invalid order info in buf:%d", pTSBuf->tsOrder); tsBufDestroy(pTSBuf); return NULL; } - + size_t infoSize = sizeof(STSGroupBlockInfo) * pTSBuf->numOfGroups; - + STSGroupBlockInfo* buf = (STSGroupBlockInfo*)calloc(1, infoSize); if (buf == NULL) { tsBufDestroy(pTSBuf); - return NULL; - } - - //int64_t pos = ftell(pTSBuf->f); //pos not used - sz = fread(buf, infoSize, 1, pTSBuf->f); + return NULL; + } + + // int64_t pos = ftell(pTSBuf->pFile); //pos not used + sz = taosReadFile(pTSBuf->pFile, buf, infoSize); UNUSED(sz); - + // the length value for each vnode is not kept in file, so does not set the length value for (int32_t i = 0; i < pTSBuf->numOfGroups; ++i) { STSGroupBlockInfoEx* pBlockList = &pTSBuf->pData[i]; memcpy(&pBlockList->info, &buf[i], sizeof(STSGroupBlockInfo)); } free(buf); - - ret = fseek(pTSBuf->f, 0, SEEK_END); + + ret = taosLSeekFile(pTSBuf->pFile, 0, SEEK_END); UNUSED(ret); - - struct stat fileStat; - if (fstat(fileno(pTSBuf->f), &fileStat) != 0) { + + int64_t file_size; + if (taosFStatFile(pTSBuf->pFile, &file_size, NULL) != 0) { tsBufDestroy(pTSBuf); return NULL; } - - pTSBuf->fileSize = (uint32_t)fileStat.st_size; + + pTSBuf->fileSize = (uint32_t)file_size; tsBufResetPos(pTSBuf); - + // ascending by default pTSBuf->cur.order = TSDB_ORDER_ASC; - -// tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, fileno(pTSBuf->f), -// pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); - + + // tscDebug("create tsBuf from file:%s, fd:%d, size:%d, numOfGroups:%d, autoDelete:%d", pTSBuf->path, + // fileno(pTSBuf->pFile), + // pTSBuf->fileSize, pTSBuf->numOfGroups, pTSBuf->autoDelete); + return pTSBuf; } @@ -148,22 +165,22 @@ void* tsBufDestroy(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return NULL; } - + tfree(pTSBuf->assistBuf); tfree(pTSBuf->tsData.rawBuf); - + tfree(pTSBuf->pData); tfree(pTSBuf->block.payload); if (!pTSBuf->remainOpen) { - fclose(pTSBuf->f); + taosCloseFile(&pTSBuf->pFile); } - + if (pTSBuf->autoDelete) { -// ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); + // ("tsBuf %p destroyed, delete tmp file:%s", pTSBuf, pTSBuf->path); remove(pTSBuf->path); } else { -// tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); + // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } taosVariantDestroy(&pTSBuf->block.tag); @@ -173,7 +190,7 @@ void* tsBufDestroy(STSBuf* pTSBuf) { static STSGroupBlockInfoEx* tsBufGetLastGroupInfo(STSBuf* pTSBuf) { int32_t last = pTSBuf->numOfGroups - 1; - + assert(last >= 0); return &pTSBuf->pData[last]; } @@ -182,40 +199,40 @@ static STSGroupBlockInfoEx* addOneGroupInfo(STSBuf* pTSBuf, int32_t id) { if (pTSBuf->numOfAlloc <= pTSBuf->numOfGroups) { uint32_t newSize = (uint32_t)(pTSBuf->numOfAlloc * 1.5); assert((int32_t)newSize > pTSBuf->numOfAlloc); - + STSGroupBlockInfoEx* tmp = (STSGroupBlockInfoEx*)realloc(pTSBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return NULL; } - + pTSBuf->pData = tmp; pTSBuf->numOfAlloc = newSize; memset(&pTSBuf->pData[pTSBuf->numOfGroups], 0, sizeof(STSGroupBlockInfoEx) * (newSize - pTSBuf->numOfGroups)); } - + if (pTSBuf->numOfGroups > 0) { STSGroupBlockInfoEx* pPrevBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pPrevBlockInfoEx->info); } - + // set initial value for vnode block STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[pTSBuf->numOfGroups].info; pBlockInfo->id = id; pBlockInfo->offset = pTSBuf->fileSize; assert(pBlockInfo->offset >= getDataStartOffset()); - + // update vnode info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups, pBlockInfo); - + // add one vnode info pTSBuf->numOfGroups += 1; - + // update the header info STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; - + STSBufUpdateHeader(pTSBuf, &header); return tsBufGetLastGroupInfo(pTSBuf); } @@ -224,7 +241,7 @@ static void shrinkBuffer(STSList* ptsData) { // shrink tmp buffer size if it consumes too many memory compared to the pre-defined size if (ptsData->allocSize >= ptsData->threshold * 2) { char* rawBuf = realloc(ptsData->rawBuf, MEM_BUF_SIZE); - if(rawBuf) { + if (rawBuf) { ptsData->rawBuf = rawBuf; ptsData->allocSize = MEM_BUF_SIZE; } @@ -244,18 +261,17 @@ static void writeDataToDisk(STSBuf* pTSBuf) { if (pTSBuf->tsData.len == 0) { return; } - + STSBlock* pBlock = &pTSBuf->block; STSList* pTsData = &pTSBuf->tsData; pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; - pBlock->compLen = - tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, - TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - - int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); + pBlock->compLen = tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len / TSDB_KEYSIZE, pBlock->payload, + pTsData->allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + + int64_t r = taosLSeekFile(pTSBuf->pFile, pTSBuf->fileSize, SEEK_SET); assert(r == 0); - + /* * format for output data: * 1. tags, number of ts, size after compressed, payload, size after compressed @@ -264,42 +280,42 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * both side has the compressed length is used to support load data forwards/backwords. */ int32_t metaLen = 0; - metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); float tfloat = (float)pBlock->tag.d; - metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.i, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); } else { trueLen = 0; - metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); } - fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); + taosWriteFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); + taosWriteFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); - metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)taosWriteFile(pTSBuf->pFile, &trueLen, sizeof(pBlock->tag.nLen)); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; - + pTSBuf->tsData.len = 0; - + STSGroupBlockInfoEx* pGroupBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + pGroupBlockInfoEx->info.compLen += blockSize; pGroupBlockInfoEx->info.numOfBlocks += 1; - + shrinkBuffer(&pTSBuf->tsData); } @@ -310,7 +326,7 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) { if (tmp == NULL) { // todo } - + ptsData->rawBuf = tmp; ptsData->allocSize = newSize; } @@ -320,8 +336,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { STSBlock* pBlock = &pTSBuf->block; // clear the memory buffer - pBlock->compLen = 0; - pBlock->padding = 0; + pBlock->compLen = 0; + pBlock->padding = 0; pBlock->numOfElem = 0; int32_t offset = -1; @@ -331,21 +347,21 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block */ - int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); - int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); - size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); - sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); - UNUSED(sz); + int32_t prev = -(int32_t)(sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); + int32_t ret = taosLSeekFile(pTSBuf->pFile, prev, SEEK_CUR); + size_t sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); + UNUSED(sz); pBlock->compLen = pBlock->padding; offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - ret = fseek(pTSBuf->f, -offset, SEEK_CUR); + ret = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(ret); } - int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nType, sizeof(pBlock->tag.nType)); + ret = taosReadFile(pTSBuf->pFile, &pBlock->tag.nLen, sizeof(pBlock->tag.nLen)); // NOTE: mix types tags are not supported size_t sz = 0; @@ -356,36 +372,36 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { memset(tp, 0, pBlock->tag.nLen + 1); pBlock->tag.pz = tp; - sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->tag.pz, (size_t)pBlock->tag.nLen); UNUSED(sz); } else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) { float tfloat = 0; - sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &tfloat, (size_t)pBlock->tag.nLen); pBlock->tag.d = (double)tfloat; UNUSED(sz); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value - sz = fread(&pBlock->tag.i, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { // TODO check the return value + sz = taosReadFile(pTSBuf->pFile, &pBlock->tag.i, (size_t)pBlock->tag.nLen); UNUSED(sz); } - sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->numOfElem, sizeof(pBlock->numOfElem)); UNUSED(sz); - sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->compLen, sizeof(pBlock->compLen)); UNUSED(sz); - sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, pBlock->payload, (size_t)pBlock->compLen); if (decomp) { pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); } - + // read the comp length at the length of comp block - sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &pBlock->padding, sizeof(pBlock->padding)); assert(pBlock->padding == pBlock->compLen); int32_t n = 0; - sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + sz = taosReadFile(pTSBuf->pFile, &n, sizeof(pBlock->tag.nLen)); if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { assert(n == 0); } else { @@ -393,24 +409,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { } UNUSED(sz); - + // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); + int32_t r = taosLSeekFile(pTSBuf->pFile, -offset, SEEK_CUR); UNUSED(r); } - + return pBlock; } // set the order of ts buffer if the ts order has not been set yet static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { STSList* ptsData = &pTSBuf->tsData; - + if (pTSBuf->tsOrder == -1) { if (ptsData->len > 0) { TSKEY lastKey = *(TSKEY*)(ptsData->rawBuf + ptsData->len - TSDB_KEYSIZE); - + if (lastKey > *(TSKEY*)pData) { pTSBuf->tsOrder = TSDB_ORDER_DESC; } else { @@ -420,7 +436,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { // no data in current vnode, more than one ts is added, check the orders TSKEY k1 = *(TSKEY*)(pData); TSKEY k2 = *(TSKEY*)(pData + TSDB_KEYSIZE); - + if (k1 < k2) { pTSBuf->tsOrder = TSDB_ORDER_ASC; } else if (k1 > k2) { @@ -432,23 +448,23 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { } else { // todo the timestamp order is set, check the asc/desc order of appended data } - + return TSDB_CODE_SUCCESS; } void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len) { STSGroupBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; - + if (pTSBuf->numOfGroups == 0 || tsBufGetLastGroupInfo(pTSBuf)->info.id != id) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); - + pBlockInfo = addOneGroupInfo(pTSBuf, id); } else { pBlockInfo = tsBufGetLastGroupInfo(pTSBuf); } - + assert(pBlockInfo->info.id == id); if ((taosVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { @@ -460,22 +476,22 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, i taosVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); - + // todo check return value setCheckTSOrder(pTSBuf, pData, len); - + ptsData->len += len; pBlockInfo->len += len; - + pTSBuf->numOfTotal += len / TSDB_KEYSIZE; - + // the size of raw data exceeds the size of the default prepared buffer, so // during getBufBlock, the output buffer needs to be large enough. if (ptsData->len >= ptsData->threshold) { writeDataToDisk(pTSBuf); shrinkBuffer(ptsData); } - + tsBufResetPos(pTSBuf); } @@ -483,15 +499,15 @@ void tsBufFlush(STSBuf* pTSBuf) { if (pTSBuf->tsData.len <= 0) { return; } - + writeDataToDisk(pTSBuf); shrinkBuffer(&pTSBuf->tsData); - + STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pTSBuf); - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, &pBlockInfoEx->info); - + // save the ts order into header STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; @@ -506,62 +522,62 @@ static int32_t tsBufFindGroupById(STSGroupBlockInfoEx* pGroupInfoEx, int32_t num break; } } - + return j; } // todo opt performance by cache blocks info static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, int32_t blockIndex) { - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { return -1; } - + // sequentially read the compressed data blocks, start from the beginning of the comp data block of this vnode int32_t i = 0; bool decomp = false; - + while ((i++) <= blockIndex) { if (readDataFromDisk(pTSBuf, TSDB_ORDER_ASC, decomp) == NULL) { return -1; } } - + // set the file position to be the end of previous comp block if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); - int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); + int32_t ret = taosLSeekFile(pTSBuf->pFile, -compBlockSize, SEEK_CUR); UNUSED(ret); } - + return 0; } static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSGroupBlockInfo* pBlockInfo, SVariant* tag) { bool decomp = false; - + int64_t offset = 0; if (pTSBuf->cur.order == TSDB_ORDER_ASC) { offset = pBlockInfo->offset; } else { // reversed traverse starts from the end of block offset = pBlockInfo->offset + pBlockInfo->compLen; } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - + for (int32_t i = 0; i < pBlockInfo->numOfBlocks; ++i) { if (readDataFromDisk(pTSBuf, pTSBuf->cur.order, decomp) == NULL) { return -1; } - + if (taosVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); + return (pTSBuf->cur.order == TSDB_ORDER_ASC) ? i : (pBlockInfo->numOfBlocks - (i + 1)); } } - + return -1; } @@ -570,14 +586,14 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex if (pBlockInfo->numOfBlocks <= blockIndex) { assert(false); } - + STSCursor* pCur = &pTSBuf->cur; if (pCur->vgroupIndex == groupIndex && ((pCur->blockIndex <= blockIndex && pCur->order == TSDB_ORDER_ASC) || - (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { + (pCur->blockIndex >= blockIndex && pCur->order == TSDB_ORDER_DESC))) { int32_t i = 0; bool decomp = false; int32_t step = abs(blockIndex - pCur->blockIndex); - + while ((++i) <= step) { if (readDataFromDisk(pTSBuf, pCur->order, decomp) == NULL) { return; @@ -588,11 +604,11 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex assert(false); } } - + STSBlock* pBlock = &pTSBuf->block; - + size_t s = pBlock->numOfElem * TSDB_KEYSIZE; - + /* * In order to accommodate all the qualified data, the actual buffer size for one block with identical tags value * may exceed the maximum allowed size during *tsBufAppend* function by invoking expandBuffer function @@ -600,16 +616,16 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t groupIndex, int32_t blockIndex if (s > pTSBuf->tsData.allocSize) { expandBuffer(&pTSBuf->tsData, (int32_t)s); } - + pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); - + assert((pTSBuf->tsData.len / TSDB_KEYSIZE == pBlock->numOfElem) && (pTSBuf->tsData.allocSize >= pTSBuf->tsData.len)); - + pCur->vgroupIndex = groupIndex; pCur->blockIndex = blockIndex; - + pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } @@ -618,11 +634,11 @@ static int32_t doUpdateGroupInfo(STSBuf* pTSBuf, int64_t offset, STSGroupBlockIn return -1; } - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + if (taosLSeekFile(pTSBuf->pFile, (int32_t)offset, SEEK_SET) != 0) { return -1; } - fwrite(pVInfo, sizeof(STSGroupBlockInfo), 1, pTSBuf->f); + taosWriteFile(pTSBuf->pFile, pVInfo, sizeof(STSGroupBlockInfo)); return 0; } @@ -631,26 +647,27 @@ STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id) { if (j == -1) { return NULL; } - + return &pTSBuf->pData[j].info; } int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader) { - if ((pTSBuf->f == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { + if ((pTSBuf->pFile == NULL) || pHeader == NULL || pHeader->numOfGroup == 0 || pHeader->magic != TS_COMP_FILE_MAGIC) { return -1; } assert(pHeader->tsOrder == TSDB_ORDER_ASC || pHeader->tsOrder == TSDB_ORDER_DESC); - int32_t r = fseek(pTSBuf->f, 0, SEEK_SET); + int32_t r = taosLSeekFile(pTSBuf->pFile, 0, SEEK_SET); if (r != 0) { -// qError("fseek failed, errno:%d", errno); + // qError("fseek failed, errno:%d", errno); return -1; } - size_t ws = fwrite(pHeader, sizeof(STSBufFileHeader), 1, pTSBuf->f); - if (ws != 1) { -// qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, (int32_t)sizeof(STSBufFileHeader)); + size_t ws = taosWriteFile(pTSBuf->pFile, pHeader, sizeof(STSBufFileHeader)); + if (ws != 1) { + // qError("ts update header fwrite failed, size:%d, expected size:%d", (int32_t)ws, + // (int32_t)sizeof(STSBufFileHeader)); return -1; } return 0; @@ -660,33 +677,33 @@ bool tsBufNextPos(STSBuf* pTSBuf) { if (pTSBuf == NULL || pTSBuf->numOfGroups == 0) { return false; } - + STSCursor* pCur = &pTSBuf->cur; - + // get the first/last position according to traverse order if (pCur->vgroupIndex == -1) { if (pCur->order == TSDB_ORDER_ASC) { tsBufGetBlock(pTSBuf, 0, 0); - + if (pTSBuf->block.numOfElem == 0) { // the whole list is empty, return tsBufResetPos(pTSBuf); return false; } else { return true; } - + } else { // get the last timestamp record in the last block of the last vnode assert(pTSBuf->numOfGroups > 0); - + int32_t groupIndex = pTSBuf->numOfGroups - 1; pCur->vgroupIndex = groupIndex; - + int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); int32_t blockIndex = pBlockInfo->numOfBlocks - 1; - + tsBufGetBlock(pTSBuf, groupIndex, blockIndex); - + pCur->tsIndex = pTSBuf->block.numOfElem - 1; if (pTSBuf->block.numOfElem == 0) { tsBufResetPos(pTSBuf); @@ -696,16 +713,16 @@ bool tsBufNextPos(STSBuf* pTSBuf) { } } } - + int32_t step = pCur->order == TSDB_ORDER_ASC ? 1 : -1; - + while (1) { assert(pTSBuf->tsData.len == pTSBuf->block.numOfElem * TSDB_KEYSIZE); - + if ((pCur->order == TSDB_ORDER_ASC && pCur->tsIndex >= pTSBuf->block.numOfElem - 1) || (pCur->order == TSDB_ORDER_DESC && pCur->tsIndex <= 0)) { int32_t id = pTSBuf->pData[pCur->vgroupIndex].info.id; - + STSGroupBlockInfo* pBlockInfo = tsBufGetGroupBlockInfo(pTSBuf, id); if (pBlockInfo == NULL || (pCur->blockIndex >= pBlockInfo->numOfBlocks - 1 && pCur->order == TSDB_ORDER_ASC) || (pCur->blockIndex <= 0 && pCur->order == TSDB_ORDER_DESC)) { @@ -714,15 +731,15 @@ bool tsBufNextPos(STSBuf* pTSBuf) { pCur->vgroupIndex = -1; return false; } - + if (pBlockInfo == NULL) { return false; } - + int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; - + } else { tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex + step); break; @@ -732,7 +749,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { break; } } - + return true; } @@ -740,7 +757,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return; } - + pTSBuf->cur = (STSCursor){.tsIndex = -1, .blockIndex = -1, .vgroupIndex = -1, .order = pTSBuf->cur.order}; } @@ -749,14 +766,14 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return elem1; } - + STSCursor* pCur = &pTSBuf->cur; if (pCur != NULL && pCur->vgroupIndex < 0) { return elem1; } STSBlock* pBlock = &pTSBuf->block; - + elem1.id = pTSBuf->pData[pCur->vgroupIndex].info.id; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); elem1.tag = &pBlock->tag; @@ -775,81 +792,82 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfGroups <= 0) { return 0; } - + if (pDestBuf->numOfGroups + pSrcBuf->numOfGroups > TS_COMP_FILE_GROUP_MAX) { return -1; } - + // src can only have one vnode index assert(pSrcBuf->numOfGroups == 1); // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); - + // compared with the last vnode id - int32_t id = tsBufGetLastGroupInfo((STSBuf*) pSrcBuf)->info.id; + int32_t id = tsBufGetLastGroupInfo((STSBuf*)pSrcBuf)->info.id; if (id != tsBufGetLastGroupInfo(pDestBuf)->info.id) { int32_t oldSize = pDestBuf->numOfGroups; int32_t newSize = oldSize + pSrcBuf->numOfGroups; - + if (pDestBuf->numOfAlloc < newSize) { pDestBuf->numOfAlloc = newSize; - + STSGroupBlockInfoEx* tmp = realloc(pDestBuf->pData, sizeof(STSGroupBlockInfoEx) * newSize); if (tmp == NULL) { return -1; } - + pDestBuf->pData = tmp; } - + // directly copy the vnode index information memcpy(&pDestBuf->pData[oldSize], pSrcBuf->pData, (size_t)pSrcBuf->numOfGroups * sizeof(STSGroupBlockInfoEx)); - + // set the new offset value for (int32_t i = 0; i < pSrcBuf->numOfGroups; ++i) { STSGroupBlockInfoEx* pBlockInfoEx = &pDestBuf->pData[i + oldSize]; pBlockInfoEx->info.offset = (pSrcBuf->pData[i].info.offset - getDataStartOffset()) + pDestBuf->fileSize; pBlockInfoEx->info.id = id; } - + pDestBuf->numOfGroups = newSize; } else { STSGroupBlockInfoEx* pBlockInfoEx = tsBufGetLastGroupInfo(pDestBuf); - + pBlockInfoEx->len += pSrcBuf->pData[0].len; pBlockInfoEx->info.numOfBlocks += pSrcBuf->pData[0].info.numOfBlocks; pBlockInfoEx->info.compLen += pSrcBuf->pData[0].info.compLen; pBlockInfoEx->info.id = id; } - - int32_t r = fseek(pDestBuf->f, 0, SEEK_END); + + int32_t r = taosLSeekFile(pDestBuf->pFile, 0, SEEK_END); assert(r == 0); - + int64_t offset = getDataStartOffset(); int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset; - int64_t written = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); - + int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size); + if (written == -1 || written != size) { return -1; } - + pDestBuf->numOfTotal += pSrcBuf->numOfTotal; int32_t oldSize = pDestBuf->fileSize; // file meta data may be cached, close and reopen the file for accurate file size. - fclose(pDestBuf->f); - pDestBuf->f = fopen(pDestBuf->path, "rb+"); - if (pDestBuf->f == NULL) { + taosCloseFile(&pDestBuf->pFile); + // pDestBuf->pFile = fopen(pDestBuf->path, "rb+"); + pDestBuf->pFile = taosOpenFile(pDestBuf->path, TD_FILE_WRITE | TD_FILE_READ); + if (pDestBuf->pFile == NULL) { return -1; } - struct stat fileStat; - if (fstat(fileno(pDestBuf->f), &fileStat) != 0) { - return -1; + int64_t file_size; + if (taosFStatFile(pDestBuf->pFile, &file_size, NULL) != 0) { + return -1; } - pDestBuf->fileSize = (uint32_t)fileStat.st_size; + pDestBuf->fileSize = (uint32_t)file_size; assert(pDestBuf->fileSize == oldSize + size); @@ -858,33 +876,33 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t id) { STSBuf* pTSBuf = tsBufCreate(true, order); - + STSGroupBlockInfo* pBlockInfo = &(addOneGroupInfo(pTSBuf, 0)->info); pBlockInfo->numOfBlocks = numOfBlocks; pBlockInfo->compLen = len; pBlockInfo->offset = getDataStartOffset(); pBlockInfo->id = id; - + // update prev vnode length info in file TSBufUpdateGroupInfo(pTSBuf, pTSBuf->numOfGroups - 1, pBlockInfo); - - int32_t ret = fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET); + + int32_t ret = taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET); if (ret == -1) { -// qError("fseek failed, errno:%d", errno); + // qError("fseek failed, errno:%d", errno); tsBufDestroy(pTSBuf); return NULL; } - size_t sz = fwrite((void*)pData, 1, len, pTSBuf->f); + size_t sz = taosWriteFile(pTSBuf->pFile, (void*)pData, len); if (sz != len) { -// qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); + // qError("ts data fwrite failed, write size:%d, expected size:%d", (int32_t)sz, len); tsBufDestroy(pTSBuf); return NULL; } pTSBuf->fileSize += len; - + pTSBuf->tsOrder = order; assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - + STSBufFileHeader header = { .magic = TS_COMP_FILE_MAGIC, .numOfGroup = pTSBuf->numOfGroups, .tsOrder = pTSBuf->tsOrder}; if (STSBufUpdateHeader(pTSBuf, &header) < 0) { @@ -893,42 +911,42 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ } // TODO taosFsync?? -// if (taosFsync(fileno(pTSBuf->f)) == -1) { -//// qError("fsync failed, errno:%d", errno); -// tsBufDestroy(pTSBuf); -// return NULL; -// } - + // if (taosFsync(fileno(pTSBuf->pFile)) == -1) { + //// qError("fsync failed, errno:%d", errno); + // tsBufDestroy(pTSBuf); + // return NULL; + // } + return pTSBuf; } STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag) { STSElem elem = {.id = -1}; - + if (pTSBuf == NULL) { return elem; } - + int32_t j = tsBufFindGroupById(pTSBuf->pData, pTSBuf->numOfGroups, id); if (j == -1) { return elem; } - + // for debug purpose // tsBufDisplay(pTSBuf); - + STSCursor* pCur = &pTSBuf->cur; STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[j].info; - + int32_t blockIndex = tsBufFindBlockByTag(pTSBuf, pBlockInfo, tag); if (blockIndex < 0) { return elem; } - + pCur->vgroupIndex = j; pCur->blockIndex = blockIndex; tsBufGetBlock(pTSBuf, j, blockIndex); - + return tsBufGetElem(pTSBuf); } @@ -937,7 +955,7 @@ STSCursor tsBufGetCursor(STSBuf* pTSBuf) { if (pTSBuf == NULL) { return c; } - + return pTSBuf->cur; } @@ -945,12 +963,12 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { if (pTSBuf == NULL || pCur == NULL) { return; } - + // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); if (pCur->vgroupIndex != -1) { tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); } - + pTSBuf->cur = *pCur; } @@ -958,7 +976,7 @@ void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order) { if (pTSBuf == NULL) { return; } - + pTSBuf->cur.order = order; } @@ -975,19 +993,19 @@ STSBuf* tsBufClone(STSBuf* pTSBuf) { void tsBufDisplay(STSBuf* pTSBuf) { printf("-------start of ts comp file-------\n"); printf("number of vnode:%d\n", pTSBuf->numOfGroups); - + int32_t old = pTSBuf->cur.order; pTSBuf->cur.order = TSDB_ORDER_ASC; - + tsBufResetPos(pTSBuf); - + while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { printf("%d-%" PRId64 "-%" PRId64 "\n", elem.id, elem.tag->i, elem.ts); } } - + pTSBuf->cur.order = old; printf("-------end of ts comp file-------\n"); } @@ -1004,36 +1022,36 @@ static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { const int32_t INITIAL_GROUPINFO_SIZE = 4; - + pTSBuf->numOfAlloc = INITIAL_GROUPINFO_SIZE; pTSBuf->pData = calloc(pTSBuf->numOfAlloc, sizeof(STSGroupBlockInfoEx)); if (pTSBuf->pData == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->tsData.rawBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->tsData.rawBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->bufSize = MEM_BUF_SIZE; pTSBuf->tsData.threshold = MEM_BUF_SIZE; pTSBuf->tsData.allocSize = MEM_BUF_SIZE; - + pTSBuf->assistBuf = malloc(MEM_BUF_SIZE); if (pTSBuf->assistBuf == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->block.payload = malloc(MEM_BUF_SIZE); if (pTSBuf->block.payload == NULL) { tsBufDestroy(pTSBuf); return NULL; } - + pTSBuf->fileSize += getDataStartOffset(); return pTSBuf; } @@ -1059,28 +1077,28 @@ void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id) { (*id) = malloc(tsBufGetNumOfGroup(pTSBuf) * sizeof(int32_t)); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { (*id)[i] = pTSBuf->pData[i].info.id; } } int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t groupIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { assert(groupIndex >= 0 && groupIndex < pTSBuf->numOfGroups); - STSGroupBlockInfo *pBlockInfo = &pTSBuf->pData[groupIndex].info; + STSGroupBlockInfo* pBlockInfo = &pTSBuf->pData[groupIndex].info; *len = 0; *numOfBlocks = 0; - if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// qError("%p: fseek failed: %s", pSql, tstrerror(code)); + if (taosLSeekFile(pTSBuf->pFile, pBlockInfo->offset, SEEK_SET) != 0) { + int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); + // qError("%p: fseek failed: %s", pSql, tstrerror(code)); return code; } - size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + size_t s = taosReadFile(pTSBuf->pFile, buf, pBlockInfo->compLen); if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); -// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + int32_t code = TAOS_SYSTEM_ERROR(taosGetErrorFile(pTSBuf->pFile)); + // tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); return code; } @@ -1103,6 +1121,4 @@ STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag) { return el; } -bool tsBufIsValidElem(STSElem* pElem) { - return pElem->id >= 0; -} +bool tsBufIsValidElem(STSElem* pElem) { return pElem->id >= 0; } diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index ee32a209203ace3e7b484c5729882d4ab57f8eab..f32698c98ad7179f2661c85fec2df3906aa76dc0 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "ttypes.h" -#include "../../../include/client/taos.h" -#include "os.h" #include "tcompression.h" const int32_t TYPE_BYTES[15] = { diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 27f1d4947d999fa5e183b9283e82be719cb3eb01..ceef12e986105ad42b1322db26c20a19a760f740 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -12,14 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "taos.h" -#include "taosdef.h" +#define _DEFAULT_SOURCE +#include "tvariant.h" #include "ttime.h" #include "ttokendef.h" -#include "ttypes.h" -#include "tutil.h" #include "tvariant.h" #define SET_EXT_INFO(converted, res, minv, maxv, exti) \ diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index e9e8d086b3c75cb97c2bbb9a14b0497cc2632b62..4821d60875f9f3daaf7e5a2fa0a66f1bc5cd0af3 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,4 +1,4 @@ -#include +#include "tcommon.h" #include #include #include @@ -162,7 +162,7 @@ TEST(testCase, Datablock_test) { ASSERT_EQ(colDataGetNumOfCols(b), 2); ASSERT_EQ(colDataGetNumOfRows(b), 40); - char* pData = colDataGet(p1, 3); + char* pData = colDataGetData(p1, 3); printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); diff --git a/source/dnode/mgmt/daemon/CMakeLists.txt b/source/dnode/mgmt/daemon/CMakeLists.txt index f1ce726d859854217f723ceca196be5ee14434ae..e07c15c95a3bc9b4fee58255a9174472732684fe 100644 --- a/source/dnode/mgmt/daemon/CMakeLists.txt +++ b/source/dnode/mgmt/daemon/CMakeLists.txt @@ -1,8 +1,9 @@ aux_source_directory(src DAEMON_SRC) add_executable(taosd ${DAEMON_SRC}) -target_link_libraries( + +target_include_directories( taosd - PUBLIC dnode - PUBLIC util - PUBLIC os + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +target_link_libraries(taosd dnode util os) diff --git a/source/dnode/mgmt/daemon/inc/dmnInt.h b/source/dnode/mgmt/daemon/inc/dmnInt.h new file mode 100644 index 0000000000000000000000000000000000000000..8a571352f021ab53f3405e11114257b7ecbd4fd8 --- /dev/null +++ b/source/dnode/mgmt/daemon/inc/dmnInt.h @@ -0,0 +1,41 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DMN_INT_H_ +#define _TD_DMN_INT_H_ + +#include "tconfig.h" +#include "dnode.h" +#include "taoserror.h" +#include "tglobal.h" +#include "tlog.h" +#include "version.h" + +#ifdef __cplusplus +extern "C" { +#endif + +SDnodeObjCfg dmnGetObjCfg(); + +void dmnDumpCfg(); +void dmnPrintVersion(); +void dmnGenerateGrant(); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DMN_INT_H_*/ diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c deleted file mode 100644 index ce672bf7b858c3daec87b8b09e8fbda734933571..0000000000000000000000000000000000000000 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "dnode.h" -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "ulog.h" - -static struct { - bool stop; - bool dumpConfig; - bool generateGrant; - bool printAuth; - bool printVersion; - char configDir[PATH_MAX]; -} dmn = {0}; - -void dmnSigintHandle(int signum, void *info, void *ctx) { - uInfo("singal:%d is received", signum); - dmn.stop = true; -} - -void dmnSetSignalHandle() { - taosSetSignal(SIGTERM, dmnSigintHandle); - taosSetSignal(SIGHUP, dmnSigintHandle); - taosSetSignal(SIGINT, dmnSigintHandle); - taosSetSignal(SIGABRT, dmnSigintHandle); - taosSetSignal(SIGBREAK, dmnSigintHandle); -} - -int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); - - for (int i = 1; i < argc; ++i) { - if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= PATH_MAX) { - printf("config file path overflow"); - return -1; - } - tstrncpy(dmn.configDir, argv[i], PATH_MAX); - } else { - printf("'-c' requires a parameter, default is %s\n", configDir); - return -1; - } - } else if (strcmp(argv[i], "-C") == 0) { - dmn.dumpConfig = true; - } else if (strcmp(argv[i], "-k") == 0) { - dmn.generateGrant = true; - } else if (strcmp(argv[i], "-A") == 0) { - dmn.printAuth = true; - } else if (strcmp(argv[i], "-V") == 0) { - dmn.printVersion = true; - } else { - } - } - - return 0; -} - -void dmnGenerateGrant() { -#if 0 - grantParseParameter(); -#endif -} - -void dmnPrintVersion() { -#ifdef TD_ENTERPRISE - char *releaseName = "enterprise"; -#else - char *releaseName = "community"; -#endif - printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); - printf("gitinfo: %s\n", gitinfo); - printf("gitinfoI: %s\n", gitinfoOfInternal); - printf("builuInfo: %s\n", buildinfo); -} - -int dmnReadConfig(const char *path) { - tstrncpy(configDir, dmn.configDir, PATH_MAX); - taosInitGlobalCfg(); - taosReadGlobalLogCfg(); - - if (taosMkDir(tsLogDir) != 0) { - printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno)); - return -1; - } - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", tsLogDir); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosInitNotes() != 0) { - printf("failed to init log file\n"); - return -1; - } - - if (taosReadCfgFromFile() != 0) { - uError("failed to read config"); - return -1; - } - - if (taosCheckAndPrintCfg() != 0) { - uError("failed to check config"); - return -1; - } - - taosSetCoreDump(tsEnableCoreFile); - return 0; -} - -void dmnDumpConfig() { taosDumpGlobalCfg(); } - -void dmnWaitSignal() { - dmnSetSignalHandle(); - while (!dmn.stop) { - taosMsleep(100); - } -} - -void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { - pCfg->sver = 30000000; // 3.0.0.0 - pCfg->numOfCores = tsNumOfCores; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; - pCfg->enableTelem = 0; - tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pCfg->buildinfo, buildinfo, 64); - tstrncpy(pCfg->gitinfo, gitinfo, 48); -} - -void dmnInitObjCfg(SDnodeObjCfg *pCfg) { - pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->statusInterval = tsStatusInterval; - pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pCfg->ratioOfQueryCores = tsRatioOfQueryCores; - pCfg->maxShellConns = tsMaxShellConns; - pCfg->shellActivityTimer = tsShellActivityTimer; - pCfg->serverPort = tsServerPort; - tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); -} - -int dmnRunDnode() { - SDnodeEnvCfg envCfg = {0}; - SDnodeObjCfg objCfg = {0}; - - dnmInitEnvCfg(&envCfg); - dmnInitObjCfg(&objCfg); - - if (dndInit(&envCfg) != 0) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - SDnode *pDnode = dndCreate(&objCfg); - if (pDnode == NULL) { - uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); - return -1; - } - - uInfo("Started TDengine service successfully."); - dmnWaitSignal(); - uInfo("TDengine is shut down!"); - - dndClose(pDnode); - dndCleanup(); - taosCloseLog(); - return 0; -} - -int main(int argc, char const *argv[]) { - if (dmnParseOption(argc, argv) != 0) { - return -1; - } - - if (dmn.generateGrant) { - dmnGenerateGrant(); - return 0; - } - - if (dmn.printVersion) { - dmnPrintVersion(); - return 0; - } - - if (dmnReadConfig(dmn.configDir) != 0) { - return -1; - } - - if (dmn.dumpConfig) { - dmnDumpConfig(); - return 0; - } - - return dmnRunDnode(); -} diff --git a/source/dnode/mgmt/daemon/src/dmnCfg.c b/source/dnode/mgmt/daemon/src/dmnCfg.c new file mode 100644 index 0000000000000000000000000000000000000000..afb9ffbdd0b43cf22e8fdcacf4eabb8156faf2d3 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnCfg.c @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" +#include "tconfig.h" + +SDnodeObjCfg dmnGetObjCfg() { + SConfig *pCfg = taosGetCfg(); + SDnodeObjCfg objCfg = {0}; + + objCfg.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; + tstrncpy(objCfg.dataDir, tsDataDir, sizeof(objCfg.dataDir)); + tstrncpy(objCfg.firstEp, tsFirst, sizeof(objCfg.firstEp)); + tstrncpy(objCfg.secondEp, tsSecond, sizeof(objCfg.firstEp)); + objCfg.serverPort = tsServerPort; + tstrncpy(objCfg.localFqdn, tsLocalFqdn, sizeof(objCfg.localFqdn)); + snprintf(objCfg.localEp, sizeof(objCfg.localEp), "%s:%u", objCfg.localFqdn, objCfg.serverPort); + objCfg.pDisks = tsDiskCfg; + objCfg.numOfDisks = tsDiskCfgNum; + return objCfg; +} + +void dmnDumpCfg() { + SConfig *pCfg = taosGetCfg(); + cfgDumpCfg(pCfg, 0, 1); +} \ No newline at end of file diff --git a/source/dnode/mgmt/daemon/src/dmnMain.c b/source/dnode/mgmt/daemon/src/dmnMain.c new file mode 100644 index 0000000000000000000000000000000000000000..df705898ca4532759ff083810f98546f2af206f8 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnMain.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +static struct { + bool stop; + bool dumpConfig; + bool generateGrant; + bool printAuth; + bool printVersion; + char envFile[PATH_MAX]; + char apolloUrl[PATH_MAX]; +} dmn = {0}; + +static void dmnSigintHandle(int signum, void *info, void *ctx) { + uInfo("singal:%d is received", signum); + dmn.stop = true; +} + +static void dmnSetSignalHandle() { + taosSetSignal(SIGTERM, dmnSigintHandle); + taosSetSignal(SIGHUP, dmnSigintHandle); + taosSetSignal(SIGINT, dmnSigintHandle); + taosSetSignal(SIGABRT, dmnSigintHandle); + taosSetSignal(SIGBREAK, dmnSigintHandle); +} + +static void dmnWaitSignal() { + dmnSetSignalHandle(); + while (!dmn.stop) { + taosMsleep(100); + } +} + +static int32_t dmnParseOption(int32_t argc, char const *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } else if (strcmp(argv[i], "-C") == 0) { + dmn.dumpConfig = true; + } else if (strcmp(argv[i], "-k") == 0) { + dmn.generateGrant = true; + } else if (strcmp(argv[i], "-V") == 0) { + dmn.printVersion = true; + } else { + } + } + + return 0; +} + +int32_t dmnRunDnode() { + if (dndInit() != 0) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + SDnodeObjCfg objCfg = dmnGetObjCfg(); + SDnode *pDnode = dndCreate(&objCfg); + if (pDnode == NULL) { + uInfo("Failed to start TDengine, please check the log"); + return -1; + } + + uInfo("Started TDengine service successfully."); + dmnWaitSignal(); + uInfo("TDengine is shut down!"); + + dndClose(pDnode); + dndCleanup(); + taosCloseLog(); + taosCleanupCfg(); + return 0; +} + +int main(int argc, char const *argv[]) { + if (dmnParseOption(argc, argv) != 0) { + return -1; + } + + if (dmn.generateGrant) { + dmnGenerateGrant(); + return 0; + } + + if (dmn.printVersion) { + dmnPrintVersion(); + return 0; + } + + if (taosCreateLog("taosdlog", 1, configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (taosInitCfg(configDir, dmn.envFile, dmn.apolloUrl, NULL, 0) != 0) { + uInfo("Failed to start TDengine since read config error"); + return -1; + } + + if (dmn.dumpConfig) { + dmnDumpCfg(); + taosCleanupCfg(); + return 0; + } + + return dmnRunDnode(); +} diff --git a/source/dnode/mgmt/daemon/src/dmnUtil.c b/source/dnode/mgmt/daemon/src/dmnUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..608bc7f3e6d0167d28cfa373983b3ceaed4041d1 --- /dev/null +++ b/source/dnode/mgmt/daemon/src/dmnUtil.c @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "dmnInt.h" + +void dmnGenerateGrant() { +#if 0 + grantParseParameter(); +#endif +} + +void dmnPrintVersion() { +#ifdef TD_ENTERPRISE + char *releaseName = "enterprise"; +#else + char *releaseName = "community"; +#endif + printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version); + printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); + printf("builuInfo: %s\n", buildinfo); +} diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 9eff246323fd640aa83f49dec1958274ade2b96f..cbd5eb5827c3b12aafdd9c63170ccedc70ba4594 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -124,9 +124,8 @@ typedef struct { typedef struct SDnode { EStat stat; SDnodeObjCfg cfg; - SDnodeEnvCfg env; SDnodeDir dir; - FileFd lockFd; + TdFilePtr pLockFile; SDnodeMgmt dmgmt; SMnodeMgmt mmgmt; SQnodeMgmt qmgmt; @@ -138,11 +137,6 @@ typedef struct SDnode { SStartupReq startup; } SDnode; -typedef struct { - int8_t once; - SDnodeEnvCfg cfg; -} SDnodeEnv; - #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index 829fabd006f78daffb8da81e19d8255a7ddcbc10..417bc1e0411b4e185d7602aea9e02735ef1a1027 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -35,6 +35,7 @@ extern "C" { #include "tthread.h" #include "ttime.h" #include "tworker.h" +#include "tglobal.h" #include "dnode.h" diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index e37a1646608e4f6e2135d9adfeec8b0c700017b9..81b020c1523e8f3443dc1ee9e68c3e52716c4337 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -62,14 +62,15 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_BNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_BNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadBnodeFile(SDnode *pDnode) { PRASE_BNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteBnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 02dced53c2ecf62e24f954eeb6555fb60474fd02..9467b56e6b58982980c5668b2a95eea78b1f666a 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -25,7 +25,7 @@ #include "tfs.h" #include "wal.h" -static SDnodeEnv dndEnv = {0}; +static int8_t once = DND_ENV_INIT; EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -59,31 +59,31 @@ void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup) { pStartup->finished = (dndGetStat(pDnode) == DND_STAT_RUNNING); } -static FileFd dndCheckRunning(char *dataDir) { +static TdFilePtr dndCheckRunning(char *dataDir) { char filepath[PATH_MAX] = {0}; snprintf(filepath, sizeof(filepath), "%s/.running", dataDir); - FileFd fd = taosOpenFileCreateWriteTrunc(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - return -1; + return NULL; } - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret != 0) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to lock file:%s since %s, quit", filepath, terrstr()); - taosCloseFile(fd); - return -1; + taosCloseFile(&pFile); + return NULL; } - return fd; + return pFile; } -static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { - pDnode->lockFd = dndCheckRunning(pCfg->dataDir); - if (pDnode->lockFd < 0) { +static int32_t dndInitDir(SDnode *pDnode, SDnodeObjCfg *pCfg) { + pDnode->pLockFile = dndCheckRunning(pCfg->dataDir); + if (pDnode->pLockFile == NULL) { return -1; } @@ -137,7 +137,6 @@ static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { } memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); - memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg)); return 0; } @@ -148,10 +147,10 @@ static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.snode); tfree(pDnode->dir.bnode); - if (pDnode->lockFd >= 0) { - taosUnLockFile(pDnode->lockFd); - taosCloseFile(pDnode->lockFd); - pDnode->lockFd = 0; + if (pDnode->pLockFile != NULL) { + taosUnLockFile(pDnode->pLockFile); + taosCloseFile(&pDnode->pLockFile); + pDnode->pLockFile = NULL; } } @@ -167,7 +166,7 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { dndSetStat(pDnode, DND_STAT_INIT); - if (dndCreateImp(pDnode, pCfg) != 0) { + if (dndInitDir(pDnode, pCfg) != 0) { dError("failed to init dnode dir since %s", terrstr()); dndClose(pDnode); return NULL; @@ -177,7 +176,14 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { tstrncpy(dCfg.dir, pDnode->cfg.dataDir, TSDB_FILENAME_LEN); dCfg.level = 0; dCfg.primary = 1; - pDnode->pTfs = tfsOpen(&dCfg, 1); + SDiskCfg *pDisks = pDnode->cfg.pDisks; + int32_t numOfDisks = pDnode->cfg.numOfDisks; + if (numOfDisks <= 0 || pDisks == NULL) { + pDisks = &dCfg; + numOfDisks = 1; + } + + pDnode->pTfs = tfsOpen(pDisks, numOfDisks); if (pDnode->pTfs == NULL) { dError("failed to init tfs since %s", terrstr()); dndClose(pDnode); @@ -259,8 +265,8 @@ void dndClose(SDnode *pDnode) { dInfo("dnode object is closed, data:%p", pDnode); } -int32_t dndInit(const SDnodeEnvCfg *pCfg) { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { +int32_t dndInit() { + if (atomic_val_compare_exchange_8(&once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { terrno = TSDB_CODE_REPEAT_INIT; dError("failed to init dnode env since %s", terrstr()); return -1; @@ -283,11 +289,8 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { } SVnodeOpt vnodeOpt = { - .sver = pCfg->sver, - .timezone = pCfg->timezone, - .locale = pCfg->locale, - .charset = pCfg->charset, - .nthreads = pCfg->numOfCommitThreads, + .sver = tsVersion, + .nthreads = tsNumOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, .sendReqToDnodeFp = dndSendReqToDnode }; @@ -298,13 +301,12 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { return -1; } - memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); dInfo("dnode env is initialized"); return 0; } void dndCleanup() { - if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + if (atomic_val_compare_exchange_8(&once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { dError("dnode env is already cleaned up"); return; } @@ -325,18 +327,6 @@ void taosGetDisk() { SDiskSize diskSize = tfsGetSize(pTfs); tfsUpdateSize(&fsMeta); - tsTotalDataDirGB = (float)(fsMeta.total / unit); - tsUsedDataDirGB = (float)(fsMeta.used / unit); - tsAvailDataDirGB = (float)(fsMeta.avail / unit); - - if (taosGetDiskSize(tsLogDir, &diskSize) == 0) { - tsTotalLogDirGB = (float)(diskSize.total / unit); - tsAvailLogDirGB = (float)(diskSize.avail / unit); - } - if (taosGetDiskSize(tsTempDir, &diskSize) == 0) { - tsTotalTmpDirGB = (float)(diskSize.total / unit); - tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit); - } #endif } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 3d149a3e60ce49c6334ac76d42371d001227a7b4..8f18222ab663e27f6e022d8a080ba0147e7750fb 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -185,16 +185,16 @@ static int32_t dndReadDnodes(SDnode *pDnode) { int32_t maxLen = 256 * 1024; char *content = calloc(1, maxLen + 1); cJSON *root = NULL; - FILE *fp = NULL; - fp = fopen(pMgmt->file, "r"); - if (fp == NULL) { + // fp = fopen(pMgmt->file, "r"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", pMgmt->file); code = 0; goto PRASE_DNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", pMgmt->file); goto PRASE_DNODE_OVER; @@ -286,7 +286,7 @@ static int32_t dndReadDnodes(SDnode *pDnode) { PRASE_DNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); @@ -309,8 +309,9 @@ PRASE_DNODE_OVER: static int32_t dndWriteDnodes(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; - FILE *fp = fopen(pMgmt->file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(pMgmt->file, "w"); + TdFilePtr pFile = taosOpenFile(pMgmt->file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { dError("failed to write %s since %s", pMgmt->file, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -341,9 +342,9 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -357,23 +358,23 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); - req.sver = pDnode->env.sver; + req.sver = tsVersion; req.dver = pMgmt->dver; req.dnodeId = pMgmt->dnodeId; req.clusterId = pMgmt->clusterId; req.rebootTime = pMgmt->rebootTime; req.updateTime = pMgmt->updateTime; - req.numOfCores = pDnode->env.numOfCores; + req.numOfCores = tsNumOfCores; req.numOfSupportVnodes = pDnode->cfg.numOfSupportVnodes; memcpy(req.dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - req.clusterCfg.statusInterval = pDnode->cfg.statusInterval; + req.clusterCfg.statusInterval = tsStatusInterval; req.clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &req.clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - memcpy(req.clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); - memcpy(req.clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); - memcpy(req.clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); + memcpy(req.clusterCfg.timezone, tsTimezone, TD_TIMEZONE_LEN); + memcpy(req.clusterCfg.locale, tsLocale, TD_LOCALE_LEN); + memcpy(req.clusterCfg.charset, tsCharset, TD_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); @@ -475,7 +476,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = pDnode->cfg.statusInterval * 1000; + int32_t ms = tsStatusInterval * 1000; setThreadName("dnode-hb"); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index c6db75c0575c3cc932e002c295d5e7e2c5d06e63..e1b16a188fd0fb826e67cce952ad024229a4b0de 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -63,14 +63,15 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_MNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_MNODE_OVER; @@ -143,7 +144,7 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) { PRASE_MNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -155,8 +156,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -184,9 +186,9 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) { } len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -273,15 +275,6 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putReqToMReadQFp = dndPutMsgToMReadQ; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; - pOption->cfg.enableTelem = pDnode->env.enableTelem; - pOption->cfg.statusInterval = pDnode->cfg.statusInterval; - pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer; - pOption->cfg.timezone = pDnode->env.timezone; - pOption->cfg.charset = pDnode->env.charset; - pOption->cfg.locale = pDnode->env.locale; - pOption->cfg.gitinfo = pDnode->env.gitinfo; - pOption->cfg.buildinfo = pDnode->env.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 64545ec09f3eb55285e8bd494435fe7b582d2003..93e2209610b98e957b956c5f9caa6babe8004e32 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -62,14 +62,15 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_QNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_QNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) { PRASE_QNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -185,7 +187,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 77686a60271868ac5a6c25de1a78f0934e223f33..4906aef246ec08d8b4c6c2ee7511aaed7fdf000f 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -62,14 +62,15 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "r"); - if (fp == NULL) { + // FILE *fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_SNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_SNODE_OVER; @@ -102,7 +103,7 @@ static int32_t dndReadSnodeFile(SDnode *pDnode) { PRASE_SNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); terrno = code; return code; @@ -114,8 +115,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { char file[PATH_MAX + 20]; snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR; dError("failed to write %s since %s", file, terrstr()); return -1; @@ -130,9 +132,9 @@ static int32_t dndWriteSnodeFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); char realfile[PATH_MAX + 20]; @@ -179,7 +181,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->sver = pDnode->env.sver; + pOption->sver = tsVersion; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index b68bed8789ba776b86ab9fdc8775c152d8657f41..157bad26a6a33493920e1cbb3875f91ffbca74ec 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -95,7 +95,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_STB_META)] = dndProcessMnodeReadMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_TABLE_META)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_VGROUP_LIST)] = dndProcessMnodeReadMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_QUERY)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_KILL_CONN)] = dndProcessMnodeWriteMsg; @@ -188,7 +188,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; rpcInit.spi = 1; @@ -343,7 +343,7 @@ static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->tmgmt; dndInitMsgFp(pMgmt); - int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); + int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } @@ -354,9 +354,9 @@ static int32_t dndInitServer(SDnode *pDnode) { rpcInit.label = "D-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = pDnode->cfg.maxShellConns; + rpcInit.sessions = tsMaxShellConns; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; + rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index b82d991179a8b698b4be348a8f51ff164cb5e2d6..f20493aa7ffae8c9a2f3c2119e645bd46389f682 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -219,14 +219,15 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - fp = fopen(file, "r"); - if (fp == NULL) { + // fp = fopen(file, "r"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { dDebug("file %s not exist", file); code = 0; goto PRASE_VNODE_OVER; } - len = (int32_t)fread(content, 1, maxLen, fp); + len = (int32_t)taosReadFile(pFile, content, maxLen); if (len <= 0) { dError("failed to read %s since content is null", file); goto PRASE_VNODE_OVER; @@ -304,7 +305,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ PRASE_VNODE_OVER: if (content != NULL) free(content); if (root != NULL) cJSON_Delete(root); - if (fp != NULL) fclose(fp); + if (pFile != NULL) taosCloseFile(&pFile); return code; } @@ -315,8 +316,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(file, PATH_MAX + 20, "%s/vnodes.json.bak", pDnode->dir.vnodes); snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); - FILE *fp = fopen(file, "w"); - if (fp == NULL) { + // FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; @@ -347,9 +349,9 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); - fwrite(content, 1, len, fp); - taosFsyncFile(fileno(fp)); - fclose(fp); + taosWriteFile(pFile, content, len); + taosFsyncFile(pFile); + taosCloseFile(&pFile); free(content); terrno = 0; @@ -421,7 +423,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = pDnode->env.numOfCores; + int32_t threadNum = tsNumOfCores; #if 1 threadNum = 1; #endif @@ -874,11 +876,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = TMIN(maxFetchThreads, pDnode->env.numOfCores); - int32_t minQueryThreads = TMAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); - int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); + int32_t maxWriteThreads = TMAX(tsNumOfCores, 1); + int32_t maxSyncThreads = TMAX(tsNumOfCores / 2, 1); SQWorkerPool *pQPool = &pMgmt->queryPool; pQPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 250d563a8b3a4d617a5e2ac37a3c6fbd063ba99f..304a370bcd4af4b5dda70b5131f9aadc95994ffb 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -20,17 +20,16 @@ #include "os.h" #include "dnode.h" -#include "tconfig.h" +#include "tmsg.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" -#include "tnote.h" #include "trpc.h" #include "tthread.h" -#include "ulog.h" #include "client.h" #include "server.h" +#include "tlog.h" class Testbase { public: diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f7b5ab45997e2975a3e60765faf62939a4319262..985625b41c28d2518e13a7b01db022b4f276f890 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -25,11 +25,6 @@ void* serverLoop(void* param) { SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SDnodeObjCfg cfg = {0}; cfg.numOfSupportVnodes = 16; - cfg.statusInterval = 1; - cfg.numOfThreadsPerCore = 1; - cfg.ratioOfQueryCores = 1; - cfg.maxShellConns = 1000; - cfg.shellActivityTimer = 30; cfg.serverPort = port; strcpy(cfg.dataDir, path); snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 771c5886efe179f26a6355a10888fc8244ab235b..00a79ebfd9b2d45ce6f6e2bc7f888f4f0522edba 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -28,25 +28,19 @@ void Testbase::InitLog(const char* path) { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; taosRemoveDir(path); taosMkDir(path); - - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path, PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } } void Testbase::Init(const char* path, int16_t port) { - SDnodeEnvCfg cfg = {0}; - cfg.numOfCommitThreads = 1; - cfg.numOfCores = 1; - dndInit(&cfg); + dndInit(); char fqdn[] = "localhost"; char firstEp[TSDB_EP_LEN] = {0}; diff --git a/source/dnode/mnode/impl/inc/mndInfoSchema.h b/source/dnode/mnode/impl/inc/mndInfoSchema.h new file mode 100644 index 0000000000000000000000000000000000000000..7db41254027c3c2924e094ae806bbcc8907f0981 --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndInfoSchema.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_INFO_SCHEMA_H_ +#define _TD_MND_INFO_SCHEMA_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SInfosTableSchema { + char *name; + int32_t type; + int32_t bytes; +} SInfosTableSchema; + +typedef struct SInfosTableMeta { + char *name; + const SInfosTableSchema *schema; + int32_t colNum; +} SInfosTableMeta; + +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); +int32_t mndInitInfos(SMnode *pMnode); +void mndCleanupInfos(SMnode *pMnode); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_INFO_SCHEMA_H_*/ diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 97cd6caf0476e4a275383d463caaf9d392416746..54595fb10533bdd3942688a450a17b0eacf6b6b2 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,8 @@ #include "tqueue.h" #include "ttime.h" #include "wal.h" +#include "version.h" +#include "tglobal.h" #ifdef __cplusplus extern "C" { @@ -56,12 +58,9 @@ typedef struct { } SProfileMgmt; typedef struct { - int8_t enable; - pthread_mutex_t lock; - pthread_cond_t cond; - volatile int32_t exit; - pthread_t thread; - char email[TSDB_FQDN_LEN]; + bool enable; + SRWLatch lock; + char email[TSDB_FQDN_LEN]; } STelemMgmt; typedef struct { @@ -81,8 +80,8 @@ typedef struct SMnode { tmr_h timer; tmr_h transTimer; tmr_h mqTimer; + tmr_h telemTimer; char *path; - SMnodeCfg cfg; int64_t checkTime; SSdb *pSdb; SDnode *pDnode; @@ -91,6 +90,7 @@ typedef struct SMnode { SProfileMgmt profileMgmt; STelemMgmt telemMgmt; SSyncMgmt syncMgmt; + SHashObj *infosMeta; MndMsgFp msgFp[TDMT_MAX]; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a41ccd989690847c3e9e20632f372b0b5a6583a2..3410a386da2d80bafcb35db5f2d97e987f753eb3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -52,7 +52,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { SSdb *pSdb = pMnode->pSdb; SClusterObj *pCluster = sdbAcquire(pSdb, SDB_CLUSTER, &pMnode->clusterId); - if (pCluster = NULL) { + if (pCluster == NULL) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9c8a4ce586fc81db61759df6819435f1c9911680..9165fa2264725253f9a1082f47ae26ae27a0f7d4 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -937,36 +937,41 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { goto USE_DB_OVER; } - pDb = mndAcquireDb(pMnode, usedbReq.db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto USE_DB_OVER; - } + char *p = strchr(usedbReq.db, '.'); + if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { + memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); + } else { + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } - pUser = mndAcquireUser(pMnode, pReq->user); - if (pUser == NULL) { - goto USE_DB_OVER; - } + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } - if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; - } + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } - usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); - if (usedbRsp.pVgroupInfos == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; - } + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } - if (usedbReq.vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); - } + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } - memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); - usedbRsp.uid = pDb->uid; - usedbRsp.vgVersion = pDb->vgVersion; - usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); - usedbRsp.hashMethod = pDb->hashMethod; + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + } int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 39ea4b482c8a8798e5344c1972cb1844133b0320..b872e933d5909d878bc55b62e1156e76d1413901 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -22,11 +22,11 @@ #include "mndUser.h" #include "mndVgroup.h" -#define TSDB_DNODE_VER_NUMBER 1 +#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 -#define TSDB_CONFIG_OPTION_LEN 16 -#define TSDB_CONIIG_VALUE_LEN 48 -#define TSDB_CONFIG_NUMBER 8 +#define TSDB_CONFIG_OPTION_LEN 16 +#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_NUMBER 8 static const char *offlineReason[] = { "", @@ -237,7 +237,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * pMnode->cfg.statusInterval) { + if (interval > 3500 * tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -272,24 +272,24 @@ static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { } static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { - if (pCfg->statusInterval != pMnode->cfg.statusInterval) { - mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, pMnode->cfg.statusInterval); + if (pCfg->statusInterval != tsStatusInterval) { + mError("statusInterval [%d - %d] cfg inconsistent", pCfg->statusInterval, tsStatusInterval); return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { - mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, + if ((0 != strcasecmp(pCfg->timezone, tsTimezone)) && (pMnode->checkTime != pCfg->checkTime)) { + mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, tsTimezone, pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->locale, pMnode->cfg.locale)) { - mError("locale [%s - %s] cfg inconsistent", pCfg->locale, pMnode->cfg.locale); + if (0 != strcasecmp(pCfg->locale, tsLocale)) { + mError("locale [%s - %s] cfg inconsistent", pCfg->locale, tsLocale); return DND_REASON_LOCALE_NOT_MATCH; } - if (0 != strcasecmp(pCfg->charset, pMnode->cfg.charset)) { - mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, pMnode->cfg.charset); + if (0 != strcasecmp(pCfg->charset, tsCharset)) { + mError("charset [%s - %s] cfg inconsistent.", pCfg->charset, tsCharset); return DND_REASON_CHARSET_NOT_MATCH; } @@ -355,12 +355,11 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { bool needCheck = !online || dnodeChanged || reboot; if (needCheck) { - if (statusReq.sver != pMnode->cfg.sver) { + if (statusReq.sver != tsVersion) { if (pDnode != NULL) { pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; } - mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, - pMnode->cfg.sver); + mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion); terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; goto PROCESS_STATUS_MSG_OVER; } @@ -632,13 +631,13 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); + strcpy(pSchema[cols].name, "name"); pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); + strcpy(pSchema[cols].name, "value"); pSchema[cols].bytes = pShow->bytes[cols]; cols++; @@ -659,29 +658,30 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { SMnode *pMnode = pReq->pMnode; + int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; char *pWrite; int32_t cols = 0; - cfgOpts[numOfRows] = "statusInterval"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%d", pMnode->cfg.statusInterval); - numOfRows++; + cfgOpts[totalRows] = "statusInterval"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%d", tsStatusInterval); + totalRows++; - cfgOpts[numOfRows] = "timezone"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.timezone); - numOfRows++; + cfgOpts[totalRows] = "timezone"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezone); + totalRows++; - cfgOpts[numOfRows] = "locale"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.locale); - numOfRows++; + cfgOpts[totalRows] = "locale"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsLocale); + totalRows++; - cfgOpts[numOfRows] = "charset"; - snprintf(cfgVals[numOfRows], TSDB_CONIIG_VALUE_LEN, "%s", pMnode->cfg.charset); - numOfRows++; + cfgOpts[totalRows] = "charset"; + snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsCharset); + totalRows++; - for (int32_t i = 0; i < numOfRows; i++) { + for (int32_t i = 0; i < totalRows; i++) { cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; @@ -691,6 +691,8 @@ static int32_t mndRetrieveConfigs(SMnodeMsg *pReq, SShowObj *pShow, char *data, pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfgVals[i], TSDB_CONIIG_VALUE_LEN); cols++; + + numOfRows++; } mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c new file mode 100644 index 0000000000000000000000000000000000000000..2c391e93e8ba0d6d0fd9950cfd3139215a043aed --- /dev/null +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndInfoSchema.h" + +static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "vnodes", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cores", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "offline_reason", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "end_point", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "vgroups", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "replica", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "quorum", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "days", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "keep", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "wallevel", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "comp", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "cachelast", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userIdxSchema[] = {{.name = "table_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userStbsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "dest_table", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY}, + }; +static const SInfosTableSchema userTblsSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "created_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + {.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stable_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "tid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "table_name", .bytes = 192, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, + {.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE}, + {.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; +static const SInfosTableSchema userUsersSchema[] = {{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "privilege", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, + }; +static const SInfosTableSchema vgroupsSchema[] = {{.name = "vg_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY}, + {.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + }; + +static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)}, + {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)}, + {TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)}, + {TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)}, + {TSDB_INS_TABLE_USER_DATABASE, userDBSchema, tListLen(userDBSchema)}, + {TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)}, + {TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)}, + {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, + {TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)}, + {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, + {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, + {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, + }; + + +int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) { + SSchema *schema = calloc(colNum, sizeof(SSchema)); + if (NULL == schema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + + for (int32_t i = 0; i < colNum; ++i) { + strcpy(schema[i].name, pSrc[i].name); + + schema[i].type = pSrc[i].type; + schema[i].colId = i + 1; + schema[i].bytes = pSrc[i].bytes; + } + + *pDst = schema; + + return TSDB_CODE_SUCCESS; +} + +int32_t mndInsInitMeta(SHashObj *hash) { + STableMetaRsp meta = {0}; + + strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); + meta.tableType = TSDB_NORMAL_TABLE; + meta.sversion = 1; + meta.tversion = 1; + + for (int32_t i = 0; i < tListLen(infosMeta); ++i) { + strcpy(meta.tbName, infosMeta[i].name); + meta.numOfColumns = infosMeta[i].colNum; + + if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) { + return -1; + } + + if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { + if (NULL == pMnode->infosMeta) { + terrno = TSDB_CODE_MND_NOT_READY; + return -1; + } + + STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); + if (NULL == meta) { + mError("invalid information schema table name:%s", tbName); + terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + return -1; + } + + *pRsp = *meta; + + pRsp->pSchemas = calloc(meta->numOfColumns, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pRsp->pSchemas = NULL; + return -1; + } + + memcpy(pRsp->pSchemas, meta->pSchemas, meta->numOfColumns * sizeof(SSchema)); + + return 0; +} + +int32_t mndInitInfos(SMnode *pMnode) { + pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (pMnode->infosMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return mndInsInitMeta(pMnode->infosMeta); +} + +void mndCleanupInfos(SMnode *pMnode) { + if (NULL == pMnode->infosMeta) { + return; + } + + void *pIter = taosHashIterate(pMnode->infosMeta, NULL); + while (pIter) { + STableMetaRsp *meta = (STableMetaRsp *)pIter; + + tfree(meta->pSchemas); + + pIter = taosHashIterate(pMnode->infosMeta, pIter); + } + + taosHashCleanup(pMnode->infosMeta); + pMnode->infosMeta = NULL; +} + + + diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index df892e224237e7eb5bf4d2c538397620c2259c80..e313c4d676e6e0770697ea879a0cf05271700538 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -21,6 +21,7 @@ #include "mndStb.h" #include "mndUser.h" #include "tglobal.h" +#include "version.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -62,7 +63,7 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); int32_t mndInitProfile(SMnode *pMnode) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - int32_t connCheckTime = pMnode->cfg.shellActivityTimer * 2; + int32_t connCheckTime = tsShellActivityTimer * 2; pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); if (pMgmt->cache == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -116,7 +117,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -142,7 +143,7 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) { return NULL; } - int32_t keepTime = pMnode->cfg.shellActivityTimer * 3; + int32_t keepTime = tsShellActivityTimer * 3; pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index afb338e97d7c53bbc0e99cd46db0386f26270ffb..8fd0c282e15224a051225f04d911a98d2191e0a0 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -64,7 +64,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); memcpy(showObj.payload, pReq->payload, pReq->payloadLen); - int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000; + int32_t keepTime = tsShellActivityTimer * 6 * 1000; SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int64_t), &showObj, size, keepTime); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 274ae100455c1bbd65ff746423103cbde8d8ab63..d8091dfa760bc939d60867852d74aae459aa207a 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -23,6 +23,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" #include "tname.h" #define TSDB_STB_VER_NUMBER 1 @@ -38,7 +39,7 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq); static int32_t mndProcessVCreateStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVAlterStbRsp(SMnodeMsg *pRsp); static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp); -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq); +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq); static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); @@ -58,7 +59,7 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_CREATE_STB_RSP, mndProcessVCreateStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndProcessVAlterStbRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndProcessVDropStbRsp); - mndSetMsgHandle(pMnode, TDMT_MND_STB_META, mndProcessStbMetaReq); + mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_STB, mndGetStbMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); @@ -1310,7 +1311,7 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { +static int32_t mndProcessTableMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; int32_t code = -1; STableInfoReq infoReq = {0}; @@ -1321,9 +1322,16 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { goto RETRIEVE_META_OVER; } - mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { - goto RETRIEVE_META_OVER; + if (0 == strcmp(infoReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + mDebug("information_schema table:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildInsTableSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } + } else { + mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; + } } int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); @@ -1553,4 +1561,4 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 static void mndCancelGetNextStb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndTelem.c b/source/dnode/mnode/impl/src/mndTelem.c index 5beb1b10e36bc954b9283113b91cf7ca4ab4aa01..d6a4c76c6220b8b0cc796a48497ea45141c5a4d1 100644 --- a/source/dnode/mnode/impl/src/mndTelem.c +++ b/source/dnode/mnode/impl/src/mndTelem.c @@ -21,8 +21,8 @@ #include "tversion.h" #define TELEMETRY_SERVER "telemetry.taosdata.com" -#define TELEMETRY_PORT 80 -#define REPORT_INTERVAL 86400 +#define TELEMETRY_PORT 80 +#define REPORT_INTERVAL 86400 static void mndBeginObject(SBufferWriter* bw) { tbufWriteChar(bw, '{'); } @@ -33,24 +33,7 @@ static void mndCloseObject(SBufferWriter* bw) { } else { tbufWriteChar(bw, '}'); } - tbufWriteChar(bw, ','); -} - -#if 0 -static void beginArray(SBufferWriter* bw) { - tbufWriteChar(bw, '['); -} - -static void closeArray(SBufferWriter* bw) { - size_t len = tbufTell(bw); - if (tbufGetData(bw, false)[len - 1] == ',') { - tbufWriteCharAt(bw, len - 1, ']'); - } else { - tbufWriteChar(bw, ']'); - } - tbufWriteChar(bw, ','); } -#endif static void mndWriteString(SBufferWriter* bw, const char* str) { tbufWriteChar(bw, '"'); @@ -61,7 +44,7 @@ static void mndWriteString(SBufferWriter* bw, const char* str) { static void mndAddIntField(SBufferWriter* bw, const char* k, int64_t v) { mndWriteString(bw, k); tbufWriteChar(bw, ':'); - char buf[32]; + char buf[32] = {0}; sprintf(buf, "%" PRId64, v); tbufWrite(bw, buf, strlen(buf)); tbufWriteChar(bw, ','); @@ -79,12 +62,13 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { size_t size = 0; int32_t done = 0; - FILE* fp = fopen("/proc/cpuinfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/cpuinfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { return; } - while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) { + while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { const char* v = strchr(line, ':') + 2; @@ -100,20 +84,21 @@ static void mndAddCpuInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/etc/os-release", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/etc/os-release", "r"); + TdFilePtr pFile = taosOpenFile("/etc/os-release", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "PRETTY_NAME", 11) == 0) { const char* p = strchr(line, '=') + 1; @@ -126,20 +111,21 @@ static void mndAddOsInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { char* line = NULL; size_t size = 0; - FILE* fp = fopen("/proc/meminfo", "r"); - if (fp == NULL) { + // FILE* fp = fopen("/proc/meminfo", "r"); + TdFilePtr pFile = taosOpenFile("/proc/meminfo", TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { return; } - while ((size = tgetline(&line, &size, fp)) != -1) { + while ((size = taosGetLineFile(pFile, &line)) != -1) { line[size - 1] = '\0'; if (strncmp(line, "MemTotal", 8) == 0) { const char* p = strchr(line, ':') + 1; @@ -149,19 +135,15 @@ static void mndAddMemoryInfo(SMnode* pMnode, SBufferWriter* bw) { } } - free(line); - fclose(fp); + if(line != NULL) free(line); + taosCloseFile(&pFile); } static void mndAddVersionInfo(SMnode* pMnode, SBufferWriter* bw) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char vstr[32] = {0}; - taosVersionIntToStr(pMnode->cfg.sver, vstr, 32); - - mndAddStringField(bw, "version", vstr); - mndAddStringField(bw, "buildInfo", pMnode->cfg.buildinfo); - mndAddStringField(bw, "gitInfo", pMnode->cfg.gitinfo); + mndAddStringField(bw, "version", version); + mndAddStringField(bw, "buildInfo", buildinfo); + mndAddStringField(bw, "gitInfo", gitinfo); mndAddStringField(bw, "email", pMgmt->email); } @@ -184,24 +166,17 @@ static void mndAddRuntimeInfo(SMnode* pMnode, SBufferWriter* bw) { } static void mndSendTelemetryReport(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - - char buf[128] = {0}; - uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); - if (ip == 0xffffffff) { - mDebug("failed to get IP address of " TELEMETRY_SERVER " since :%s", strerror(errno)); - return; - } - SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); - if (fd < 0) { - mDebug("failed to create socket for telemetry, reason:%s", strerror(errno)); - return; - } + STelemMgmt* pMgmt = &pMnode->telemMgmt; + SBufferWriter bw = tbufInitWriter(NULL, false); + int32_t code = -1; + char buf[128] = {0}; + SOCKET fd = 0; char clusterName[64] = {0}; - mndGetClusterName(pMnode, clusterName, sizeof(clusterName)); + if (mndGetClusterName(pMnode, clusterName, sizeof(clusterName)) != 0) { + goto SEND_OVER; + } - SBufferWriter bw = tbufInitWriter(NULL, false); mndBeginObject(&bw); mndAddStringField(&bw, "instanceId", clusterName); mndAddIntField(&bw, "reportVersion", 1); @@ -212,111 +187,95 @@ static void mndSendTelemetryReport(SMnode* pMnode) { mndAddRuntimeInfo(pMnode, &bw); mndCloseObject(&bw); + uint32_t ip = taosGetIpv4FromFqdn(TELEMETRY_SERVER); + if (ip == 0xffffffff) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to get ip of %s since :%s", TELEMETRY_SERVER, terrstr()); + goto SEND_OVER; + } + + fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0); + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to create socket to %s:%d since:%s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + goto SEND_OVER; + } + const char* header = "POST /report HTTP/1.1\n" "Host: " TELEMETRY_SERVER "\n" "Content-Type: application/json\n" "Content-Length: "; + if (taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)) < 0) { + goto SEND_OVER; + } - taosWriteSocket(fd, (void*)header, (int32_t)strlen(header)); - int32_t contLen = (int32_t)(tbufTell(&bw) - 1); + int32_t contLen = (int32_t)(tbufTell(&bw)); sprintf(buf, "%d\n\n", contLen); - taosWriteSocket(fd, buf, (int32_t)strlen(buf)); - taosWriteSocket(fd, tbufGetData(&bw, false), contLen); - tbufCloseWriter(&bw); + if (taosWriteSocket(fd, buf, (int32_t)strlen(buf)) < 0) { + goto SEND_OVER; + } + + const char* pCont = tbufGetData(&bw, false); + if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) { + goto SEND_OVER; + } // read something to avoid nginx error 499 if (taosReadSocket(fd, buf, 10) < 0) { - mDebug("failed to receive response since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + mError("failed to receive response since %s", terrstr()); + goto SEND_OVER; } + mInfo("send telemetry to %s:%d, len:%d content: %s", TELEMETRY_SERVER, TELEMETRY_PORT, contLen, pCont); + code = 0; + +SEND_OVER: + tbufCloseWriter(&bw); taosCloseSocket(fd); + + if (code != 0) { + mError("failed to send telemetry to %s:%d since %s", TELEMETRY_SERVER, TELEMETRY_PORT, terrstr()); + } } -static void* mndTelemThreadFp(void* param) { - SMnode* pMnode = param; +static int32_t mndProcessTelemTimer(SMnodeMsg* pReq) { + SMnode* pMnode = pReq->pMnode; STelemMgmt* pMgmt = &pMnode->telemMgmt; + if (!pMgmt->enable) return 0; - struct timespec end = {0}; - clock_gettime(CLOCK_REALTIME, &end); - end.tv_sec += 300; // wait 5 minutes before send first report - - setThreadName("mnd-telem"); - - while (!pMgmt->exit) { - int32_t r = 0; - struct timespec ts = end; - pthread_mutex_lock(&pMgmt->lock); - r = pthread_cond_timedwait(&pMgmt->cond, &pMgmt->lock, &ts); - pthread_mutex_unlock(&pMgmt->lock); - if (r == 0) break; - if (r != ETIMEDOUT) continue; - - if (mndIsMaster(pMnode)) { - mndSendTelemetryReport(pMnode); - } - end.tv_sec += REPORT_INTERVAL; - } - - return NULL; + taosWLockLatch(&pMgmt->lock); + mndSendTelemetryReport(pMnode); + taosWUnLockLatch(&pMgmt->lock); + return 0; } static void mndGetEmail(SMnode* pMnode, char* filepath) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - int32_t fd = taosOpenFileRead(filepath); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { return; } - if (taosReadFile(fd, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { + if (taosReadFile(pFile, (void*)pMgmt->email, TSDB_FQDN_LEN) < 0) { mError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno)); } - taosCloseFile(fd); + taosCloseFile(&pFile); } int32_t mndInitTelem(SMnode* pMnode) { STelemMgmt* pMgmt = &pMnode->telemMgmt; - pMgmt->enable = pMnode->cfg.enableTelem; - - if (!pMgmt->enable) return 0; - - pMgmt->exit = 0; - pthread_mutex_init(&pMgmt->lock, NULL); - pthread_cond_init(&pMgmt->cond, NULL); - pMgmt->email[0] = 0; - + pMgmt->enable = tsEnableTelemetryReporting; + taosInitRWLatch(&pMgmt->lock); mndGetEmail(pMnode, "/usr/local/taos/email"); - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - - int32_t code = pthread_create(&pMgmt->thread, &attr, mndTelemThreadFp, pMnode); - pthread_attr_destroy(&attr); - if (code != 0) { - mDebug("failed to create telemetry thread since :%s", strerror(code)); - } - - mInfo("mnd telemetry is initialized"); + mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); + mDebug("mnode telemetry is initialized"); return 0; } -void mndCleanupTelem(SMnode* pMnode) { - STelemMgmt* pMgmt = &pMnode->telemMgmt; - if (!pMgmt->enable) return; - - if (taosCheckPthreadValid(pMgmt->thread)) { - pthread_mutex_lock(&pMgmt->lock); - pMgmt->exit = 1; - pthread_cond_signal(&pMgmt->cond); - pthread_mutex_unlock(&pMgmt->lock); - - pthread_join(pMgmt->thread, NULL); - } - - pthread_mutex_destroy(&pMgmt->lock); - pthread_cond_destroy(&pMgmt->cond); -} +void mndCleanupTelem(SMnode* pMnode) {} diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7d684cd5427de374dec1693f347260fc3dbc7672..a009e01a52cfd1a535529cd71c47235b000a828a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -71,7 +71,7 @@ int32_t mndInitTrans(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTransActionUpdate, .deleteFp = (SdbDeleteFp)mndTransActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_TRANS, mndProcessTransReq); + mndSetMsgHandle(pMnode, TDMT_MND_TRANS_TIMER, mndProcessTransReq); mndSetMsgHandle(pMnode, TDMT_MND_KILL_TRANS, mndProcessKillTransReq); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TRANS, mndGetTransMeta); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7cdae79c0d35db885cf521721192396d1094dd56..b1cdf484b4805a9355da48db15cf5df5ca7ea91b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -19,7 +19,7 @@ #include "mndDb.h" #include "mndShow.h" #include "mndTrans.h" -#include "tkey.h" +#include "tbase64.h" #define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 5b5bf661e54940a65550e4661678efe966ebd126..ab5d0d794b6f0416056f418dbc83d104f7362d69 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -36,6 +36,11 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndInfoSchema.h" + +#define MQ_TIMER_MS 3000 +#define TRNAS_TIMER_MS 6000 +#define TELEM_TIMER_MS 86400000 int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { if (pMnode == NULL || pMnode->sendReqToDnodeFp == NULL) { @@ -74,16 +79,16 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { return pReq; } -static void mndTransReExecute(void *param, void *tmrId) { +static void mndPullupTrans(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndTransReExecute, 3000, pMnode, pMnode->timer, &pMnode->transTimer); + taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); } static void mndCalMqRebalance(void *param, void *tmrId) { @@ -95,25 +100,39 @@ static void mndCalMqRebalance(void *param, void *tmrId) { pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } - taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer); + taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); +} + +static void mndPullupTelem(void *param, void *tmrId) { + SMnode *pMnode = param; + if (mndIsMaster(pMnode)) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); + } + + taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); } static int32_t mndInitTimer(SMnode *pMnode) { + pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); if (pMnode->timer == NULL) { - pMnode->timer = taosTmrInit(5000, 200, 3600000, "MND"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - if (pMnode->timer == NULL) { + if (taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndTransReExecute, 6000, pMnode, pMnode->timer, &pMnode->transTimer)) { + if (taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - if (taosTmrReset(mndCalMqRebalance, 3000, pMnode, pMnode->timer, &pMnode->mqTimer)) { + if (taosTmrReset(mndPullupTelem, 60000, pMnode, pMnode->timer, &pMnode->telemTimer)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -127,6 +146,8 @@ static void mndCleanupTimer(SMnode *pMnode) { pMnode->transTimer = NULL; taosTmrStop(pMnode->mqTimer); pMnode->mqTimer = NULL; + taosTmrStop(pMnode->telemTimer); + pMnode->telemTimer = NULL; taosTmrCleanUp(pMnode->timer); pMnode->timer = NULL; } @@ -201,6 +222,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-offset", mndInitOffset, mndCleanupOffset) != 0) return -1; if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (pMnode->clusterId <= 0) { @@ -270,28 +292,13 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sendReqToDnodeFp = pOption->sendReqToDnodeFp; pMnode->sendReqToMnodeFp = pOption->sendReqToMnodeFp; pMnode->sendRedirectRspFp = pOption->sendRedirectRspFp; - pMnode->cfg.sver = pOption->cfg.sver; - pMnode->cfg.enableTelem = pOption->cfg.enableTelem; - pMnode->cfg.statusInterval = pOption->cfg.statusInterval; - pMnode->cfg.shellActivityTimer = pOption->cfg.shellActivityTimer; - pMnode->cfg.timezone = strdup(pOption->cfg.timezone); - pMnode->cfg.locale = strdup(pOption->cfg.locale); - pMnode->cfg.charset = strdup(pOption->cfg.charset); - pMnode->cfg.gitinfo = strdup(pOption->cfg.gitinfo); - pMnode->cfg.buildinfo = strdup(pOption->cfg.buildinfo); if (pMnode->sendReqToDnodeFp == NULL || pMnode->sendReqToMnodeFp == NULL || pMnode->sendRedirectRspFp == NULL || - pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0 || - pMnode->cfg.statusInterval < 1) { + pMnode->putReqToMWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } - if (pMnode->cfg.timezone == NULL || pMnode->cfg.locale == NULL || pMnode->cfg.charset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - return 0; } @@ -361,11 +368,6 @@ void mndClose(SMnode *pMnode) { mDebug("start to close mnode"); mndCleanupSteps(pMnode, -1); tfree(pMnode->path); - tfree(pMnode->cfg.charset); - tfree(pMnode->cfg.locale); - tfree(pMnode->cfg.timezone); - tfree(pMnode->cfg.gitinfo); - tfree(pMnode->cfg.buildinfo); tfree(pMnode); mDebug("mnode is closed"); } @@ -406,8 +408,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { return NULL; } - if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && - pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { + if (pRpcMsg->msgType != TDMT_MND_TRANS_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && + pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE && pRpcMsg->msgType != TDMT_MND_TELEM_TIMER) { SRpcConnInfo connInfo = {0}; if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { taosFreeQitem(pMsg); @@ -422,12 +424,14 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->createdTime = taosGetTimestampSec(); - mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + if (pRpcMsg != NULL) { + mTrace("msg:%p, is created, app:%p RPC:%p user:%s", pMsg, pRpcMsg->ahandle, pRpcMsg->handle, pMsg->user); + } return pMsg; } void mndCleanupMsg(SMnodeMsg *pMsg) { - mTrace("msg:%p, is destroyed, app:%p RPC:%p", pMsg, pMsg->rpcMsg.ahandle, pMsg->rpcMsg.handle); + mTrace("msg:%p, is destroyed", pMsg); rpcFreeCont(pMsg->rpcMsg.pCont); pMsg->rpcMsg.pCont = NULL; taosFreeQitem(pMsg); @@ -501,9 +505,17 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { } } + +// Note: uid 0 is reserved uint64_t mndGenerateUid(char *name, int32_t len) { - int64_t us = taosGetTimestampUs(); int32_t hashval = MurmurHash3_32(name, len); - uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; - return x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + + do { + int64_t us = taosGetTimestampUs(); + uint64_t x = (us & 0x000000FFFFFFFFFF) << 24; + uint64_t uuid = x + ((hashval & ((1ul << 16) - 1ul)) << 8) + (taosRand() & ((1ul << 8) - 1ul)); + if (uuid) { + return uuid; + } + } while (true); } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index a3efad8aa20e57eb1d4d8910f790f077e7cc4253..17fda48cd7ec3d132a290757e1ef74367316ba3c 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -119,8 +119,8 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { EXPECT_EQ(test.GetShowRows(), 2); CheckInt32(2); CheckInt32(3); - CheckInt32(0); - CheckInt32(0); + IgnoreInt32(); + IgnoreInt32(); CheckInt16(1); CheckInt16(1); CheckBinary("master", 9); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c551097e720e32b5e303bd5180b8a9ad18edb130..f45c0795cd6441546a351a0e814f7a25c5d5ae95 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -339,7 +339,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { void* pReq = rpcMallocCont(contLen); tSerializeSTableInfoReq(pReq, contLen, &infoReq); - SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_TABLE_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 88d4fc4f750a44e3601c9170b10676159619723a..cea93017f4bbc1dd042d55a46a01561dfebd5881 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -27,25 +27,25 @@ class MndTestTrans : public ::testing::Test { static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; - FileFd fd = taosOpenFileRead(file); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); int32_t size = 3 * 1024 * 1024; void* buffer = malloc(size); - int32_t readLen = taosReadFile(fd, buffer, size); + int32_t readLen = taosReadFile(pFile, buffer, size); if (readLen < 0 || readLen == size) { ASSERT(1); } - taosCloseFile(fd); + taosCloseFile(&pFile); test.ServerStop(); - fd = taosOpenFileCreateWriteTrunc(file); - int32_t writeLen = taosWriteFile(fd, buffer, readLen); + pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + int32_t writeLen = taosWriteFile(pFile, buffer, readLen); if (writeLen < 0 || writeLen == readLen) { ASSERT(1); } free(buffer); - taosFsyncFile(fd); - taosCloseFile(fd); + taosFsyncFile(pFile); + taosCloseFile(&pFile); taosMsleep(1000); test.ServerStart(); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 6d17423324d857c511f4484ae5666310d5da5a29..a3fe4dfb14996f4e92d0a189f8f6ca07a2a6449e 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -37,8 +37,8 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) { return 0; } -static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { - int32_t ret = taosReadFile(fd, &pSdb->curVer, sizeof(int64_t)); +static int32_t sdbReadFileHead(SSdb *pSdb, TdFilePtr pFile) { + int32_t ret = taosReadFile(pFile, &pSdb->curVer, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,7 +50,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t maxId = -1; - ret = taosReadFile(fd, &maxId, sizeof(int64_t)); + ret = taosReadFile(pFile, &maxId, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -66,7 +66,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { for (int32_t i = 0; i < SDB_TABLE_SIZE; ++i) { int64_t ver = -1; - ret = taosReadFile(fd, &ver, sizeof(int64_t)); + ret = taosReadFile(pFile, &ver, sizeof(int64_t)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -81,7 +81,7 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { } char reserve[SDB_RESERVE_SIZE] = {0}; - ret = taosReadFile(fd, reserve, sizeof(reserve)); + ret = taosReadFile(pFile, reserve, sizeof(reserve)); if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -94,8 +94,8 @@ static int32_t sdbReadFileHead(SSdb *pSdb, FileFd fd) { return 0; } -static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { - if (taosWriteFile(fd, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { +static int32_t sdbWriteFileHead(SSdb *pSdb, TdFilePtr pFile) { + if (taosWriteFile(pFile, &pSdb->curVer, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -105,7 +105,7 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { maxId = pSdb->maxId[i]; } - if (taosWriteFile(fd, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &maxId, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -116,14 +116,14 @@ static int32_t sdbWriteFileHead(SSdb *pSdb, FileFd fd) { if (i < SDB_MAX) { ver = pSdb->tableVer[i]; } - if (taosWriteFile(fd, &ver, sizeof(int64_t)) != sizeof(int64_t)) { + if (taosWriteFile(pFile, &ver, sizeof(int64_t)) != sizeof(int64_t)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } char reserve[SDB_RESERVE_SIZE] = {0}; - if (taosWriteFile(fd, reserve, sizeof(reserve)) != sizeof(reserve)) { + if (taosWriteFile(pFile, reserve, sizeof(reserve)) != sizeof(reserve)) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -148,25 +148,25 @@ int32_t sdbReadFile(SSdb *pSdb) { snprintf(file, sizeof(file), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); mDebug("start to read file:%s", file); - FileFd fd = taosOpenFileRead(file); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(file, TD_FILE_READ); + if (pFile == NULL) { free(pRaw); terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, terrstr()); return 0; } - if (sdbReadFileHead(pSdb, fd) != 0) { + if (sdbReadFileHead(pSdb, pFile) != 0) { mError("failed to read file:%s head since %s", file, terrstr()); pSdb->curVer = -1; free(pRaw); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } while (1) { readLen = sizeof(SSdbRaw); - ret = taosReadFile(fd, pRaw, readLen); + ret = taosReadFile(pFile, pRaw, readLen); if (ret == 0) break; if (ret < 0) { @@ -182,7 +182,7 @@ int32_t sdbReadFile(SSdb *pSdb) { } readLen = pRaw->dataLen + sizeof(int32_t); - ret = taosReadFile(fd, pRaw->pData, readLen); + ret = taosReadFile(pFile, pRaw->pData, readLen); if (ret < 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to read file:%s since %s", file, tstrerror(code)); @@ -214,7 +214,7 @@ int32_t sdbReadFile(SSdb *pSdb) { mDebug("read file:%s successfully, ver:%" PRId64, file, pSdb->lastCommitVer); PARSE_SDB_DATA_ERROR: - taosCloseFile(fd); + taosCloseFile(&pFile); sdbFreeRaw(pRaw); terrno = code; @@ -232,16 +232,16 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, pSdb->lastCommitVer); - FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); - if (fd <= 0) { + TdFilePtr pFile = taosOpenFile(tmpfile, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); mError("failed to open file:%s for write since %s", tmpfile, terrstr()); return -1; } - if (sdbWriteFileHead(pSdb, fd) != 0) { + if (sdbWriteFileHead(pSdb, pFile) != 0) { mError("failed to write file:%s head since %s", tmpfile, terrstr()); - taosCloseFile(fd); + taosCloseFile(&pFile); return -1; } @@ -269,7 +269,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { if (pRaw != NULL) { pRaw->status = pRow->status; int32_t writeLen = sizeof(SSdbRaw) + pRaw->dataLen; - if (taosWriteFile(fd, pRaw, writeLen) != writeLen) { + if (taosWriteFile(pFile, pRaw, writeLen) != writeLen) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -277,7 +277,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } int32_t cksum = taosCalcChecksum(0, (const uint8_t *)pRaw, sizeof(SSdbRaw) + pRaw->dataLen); - if (taosWriteFile(fd, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { + if (taosWriteFile(pFile, &cksum, sizeof(int32_t)) != sizeof(int32_t)) { code = TAOS_SYSTEM_ERROR(errno); taosHashCancelIterate(hash, ppRow); sdbFreeRaw(pRaw); @@ -296,14 +296,14 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) { } if (code == 0) { - code = taosFsyncFile(fd); + code = taosFsyncFile(pFile); if (code != 0) { code = TAOS_SYSTEM_ERROR(errno); mError("failed to write file:%s since %s", tmpfile, tstrerror(code)); } } - taosCloseFile(fd); + taosCloseFile(&pFile); if (code == 0) { code = taosRenameFile(tmpfile, curfile); diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 44a352ec54966af9a1cb13c767cc8a23988395c6..c98a97d0dc19d669980a0baa2370243b22aebdf3 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -16,8 +16,7 @@ #ifndef _TD_META_H_ #define _TD_META_H_ -#include "mallocator.h" -#include "os.h" +#include "tmallocator.h" #include "tmsg.h" #include "trow.h" diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index d8c9d11ce9902d3c59621446349ecd056d4f9937..36626514ec1c3c7925295f2babb8d479c03f8467 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -16,11 +16,10 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ -#include "common.h" +#include "tcommon.h" #include "executor.h" -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" -#include "os.h" #include "scheduler.h" #include "taoserror.h" #include "tlist.h" diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 7bdd8fc266ac9900bb6e719f65de87beb2e68a06..677ca9c3367a85045318bd1d09f75d0fa725f188 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -16,9 +16,9 @@ #ifndef _TD_TSDB_H_ #define _TD_TSDB_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" -#include "common.h" +#include "tcommon.h" #include "tfs.h" #ifdef __cplusplus diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 7c8f97bb8b1e7a6126f8d4439df2a2f893864477..31f04e840ac2a82fc95dac80461cd5efb5c07d27 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -216,12 +216,15 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } + return 0; } diff --git a/source/dnode/vnode/src/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h index b76c08b7b4a9a59081e3de6eac70b8e1aedafe15..f4128f7170675633664f1cb6f6685563eaccf70c 100644 --- a/source/dnode/vnode/src/inc/metaDef.h +++ b/source/dnode/vnode/src/inc/metaDef.h @@ -16,7 +16,7 @@ #ifndef _TD_META_DEF_H_ #define _TD_META_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index a801b6f7aeb890ff567909d2b76faa96c6bb9ea6..30a83ca63446dbcf11448d5f06728353d6b97010 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -141,9 +141,9 @@ typedef struct { STqMetaList* unconnectTopic; // TODO:temporaral use, to be replaced by unified tfile - int fileFd; + TdFilePtr pFile; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + TdFilePtr pIdxFile; char* dirPath; int32_t tqConfigFlag; diff --git a/source/dnode/vnode/src/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h index 55e16218ad71fffc89306489d9b2d260ce93f4f7..98c0de32a851fdfc1d50366f77dfaf8d16303f61 100644 --- a/source/dnode/vnode/src/inc/tsdbDef.h +++ b/source/dnode/vnode/src/inc/tsdbDef.h @@ -16,7 +16,7 @@ #ifndef _TD_TSDB_DEF_H_ #define _TD_TSDB_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" #include "meta.h" #include "tcompression.h" #include "tglobal.h" diff --git a/source/dnode/vnode/src/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h index 4a79c6358dbc7191d8cb7cbd5cf631a76a15cd73..15c8d512d6d54beca9871da0021b5881e4a5a1f9 100644 --- a/source/dnode/vnode/src/inc/tsdbFile.h +++ b/source/dnode/vnode/src/inc/tsdbFile.h @@ -28,17 +28,17 @@ #define TSDB_FILE_INFO(tf) (&((tf)->info)) #define TSDB_FILE_F(tf) (&((tf)->f)) -#define TSDB_FILE_FD(tf) ((tf)->fd) +#define TSDB_FILE_PFILE(tf) ((tf)->pFile) #define TSDB_FILE_FULL_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0) +#define TSDB_FILE_OPENED(tf) (TSDB_FILE_PFILE(tf) != NULL) #define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf)) -#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1) +#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_PFILE(f) = NULL) #define TSDB_FILE_LEVEL(tf) (TSDB_FILE_F(tf)->did.level) #define TSDB_FILE_ID(tf) (TSDB_FILE_F(tf)->did.id) #define TSDB_FILE_DID(tf) (TSDB_FILE_F(tf)->did) #define TSDB_FILE_REL_NAME(tf) (TSDB_FILE_F(tf)->rname) #define TSDB_FILE_ABS_NAME(tf) (TSDB_FILE_F(tf)->aname) -#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_FD(tf)) +#define TSDB_FILE_FSYNC(tf) taosFsyncFile(TSDB_FILE_PFILE(tf)) #define TSDB_FILE_STATE(tf) ((tf)->state) #define TSDB_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) @@ -178,10 +178,10 @@ typedef struct { } SDFInfo; typedef struct { - SDFInfo info; - STfsFile f; - int fd; - uint8_t state; + SDFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; } SDFile; void tsdbInitDFile(STsdb *pRepo, SDFile* pDFile, SDiskID did, int fid, uint32_t ver, TSDB_FILE_T ftype); @@ -198,8 +198,8 @@ static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFi static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { ASSERT(!TSDB_FILE_OPENED(pDFile)); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), flags); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -209,15 +209,15 @@ static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) { static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) { if (TSDB_FILE_OPENED(pDFile)) { - close(pDFile->fd); + taosCloseFile(&pDFile->pFile); TSDB_FILE_SET_CLOSED(pDFile); } } static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) { - ASSERT(TSDB_FILE_OPENED(pDFile)); + // ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t loffset = taosLSeekFile(TSDB_FILE_FD(pDFile), offset, whence); + int64_t loffset = taosLSeekFile(TSDB_FILE_PFILE(pDFile), offset, whence); if (loffset < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -229,7 +229,7 @@ static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int wh static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nwrite = taosWriteFile(pDFile->fd, buf, nbyte); + int64_t nwrite = taosWriteFile(pDFile->pFile, buf, nbyte); if (nwrite < nbyte) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -271,7 +271,7 @@ static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsRemoveFile(T static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) { ASSERT(TSDB_FILE_OPENED(pDFile)); - int64_t nread = taosReadFile(pDFile->fd, buf, nbyte); + int64_t nread = taosReadFile(pDFile->pFile, buf, nbyte); if (nread < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 97cd4e2612c7a9202fa33f35e19e917bb6a4b719..8ecb4e4a6ebe7a8b15a7eeef8cb6d48e5eb8d889 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -22,7 +22,7 @@ #include "tsdbFile.h" #include "tskiplist.h" #include "tsdbMemory.h" -#include "common.h" +#include "tcommon.h" typedef struct SReadH SReadH; diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index fb81ddbc5ce992c699faa4cecf1d0d16a429fbcd..75088687a810ea461daaa04793005518c0e659b7 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -16,7 +16,7 @@ #ifndef _TD_VNODE_DEF_H_ #define _TD_VNODE_DEF_H_ -#include "mallocator.h" +#include "tmallocator.h" // #include "sync.h" #include "tcoding.h" #include "tfs.h" diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index e5ccd02e484349aca0f09826a928f6e9dcd59143..3c464391d2f6aa88f62e060152b5f6b182e7456e 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "db.h" #include "metaDef.h" diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index d40cc2294fe86a30a4b24fe9571ef19e6e5a5686..6d5085ee7408a095e156fe93da44162b3a1ff89b 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -34,11 +34,11 @@ static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { } } -static inline int tqSeekLastPage(int fd) { - int offset = lseek(fd, 0, SEEK_END); +static inline int64_t tqSeekLastPage(TdFilePtr pFile) { + int offset = taosLSeekFile(pFile, 0, SEEK_END); int pageNo = offset / TQ_PAGE_SIZE; int curPageOffset = pageNo * TQ_PAGE_SIZE; - return lseek(fd, curPageOffset, SEEK_SET); + return taosLSeekFile(pFile, curPageOffset, SEEK_SET); } // TODO: the struct is tightly coupled with index entry @@ -52,10 +52,10 @@ typedef struct STqIdxPageBuf { char buffer[TQ_IDX_PAGE_BODY_SIZE]; } STqIdxPageBuf; -static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { - int offset = tqSeekLastPage(fd); +static inline int tqReadLastPage(TdFilePtr pFile, STqIdxPageBuf* pBuf) { + int offset = tqSeekLastPage(pFile); int nBytes; - if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = taosReadFile(pFile, pBuf, TQ_PAGE_SIZE)) == -1) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -65,7 +65,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { } ASSERT(nBytes == 0 || nBytes == pBuf->head.writeOffset); - return lseek(fd, offset, SEEK_SET); + return taosLSeekFile(pFile, offset, SEEK_SET); } STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, @@ -95,15 +95,15 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); - int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if (idxFd < 0) { + TdFilePtr pIdxFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pIdxFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } - pMeta->idxFd = idxFd; + pMeta->pIdxFile = pIdxFile; pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; @@ -113,14 +113,14 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F strcpy(name, path); strcat(name, "/" TQ_META_NAME); - int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if (fileFd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } - pMeta->fileFd = fileFd; + pMeta->pFile = pFile; pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; @@ -136,7 +136,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F int idxRead; int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + while ((idxRead = taosReadFile(pIdxFile, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error terrno = TAOS_SYSTEM_ERROR(errno); @@ -152,7 +152,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); - lseek(fileFd, pNode->handle.offset, SEEK_SET); + taosLSeekFile(pFile, pNode->handle.offset, SEEK_SET); if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { @@ -163,7 +163,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + if (taosReadFile(pFile, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { // TODO: read error } if (serializedObj->action == TQ_ACTION_INUSE) { @@ -237,8 +237,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { // commit data and idx tqStorePersist(pMeta); ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -263,8 +263,8 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { } int32_t tqStoreDelete(STqMetaStore* pMeta) { - close(pMeta->fileFd); - close(pMeta->idxFd); + taosCloseFile(&pMeta->pFile); + taosCloseFile(&pMeta->pIdxFile); // free memory for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; @@ -302,12 +302,12 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); /*int allocatedSize = sizeof(STqSerializedHead);*/ - int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); + int offset = taosLSeekFile(pMeta->pFile, 0, SEEK_CUR); - tqReadLastPage(pMeta->idxFd, &idxBuf); + tqReadLastPage(pMeta->pIdxFile, &idxBuf); if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { - lseek(pMeta->idxFd, 0, SEEK_END); + taosLSeekFile(pMeta->pIdxFile, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } else { @@ -329,7 +329,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } - nBytes = write(pMeta->fileFd, pSHead, pSHead->ssize); + nBytes = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytes == pSHead->ssize); } @@ -340,7 +340,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } - int nBytesTxn = write(pMeta->fileFd, pSHead, pSHead->ssize); + int nBytesTxn = taosWriteFile(pMeta->pFile, pSHead, pSHead->ssize); ASSERT(nBytesTxn == pSHead->ssize); nBytes += nBytesTxn; } @@ -355,7 +355,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { idxBuf.head.writeOffset += TQ_IDX_SIZE; if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { - nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); + nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, TQ_PAGE_SIZE); // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); @@ -391,13 +391,13 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { free(pSHead); // TODO: write new version in tfile if ((char*)bufPtr != idxBuf.buffer) { - int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); + int nBytes = taosWriteFile(pMeta->pIdxFile, &idxBuf, idxBuf.head.writeOffset); // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } // TODO: using fsync in tfile - fsync(pMeta->idxFd); - fsync(pMeta->fileFd); + taosFsyncFile(pMeta->pIdxFile); + taosFsyncFile(pMeta->pFile); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 26c313f42146fddc6c7690d0bb0cf2ecba21ccdc..f7e4d56fe2c22e77895b9b6971c8a937b6277c68 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -523,7 +523,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh)); SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith); tsdbInitDFileEx(pWDataf, pRDataf); - if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWDataf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWDataf), tstrerror(terrno)); @@ -543,7 +544,8 @@ static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid tsdbInitDFileEx(pWLastf, pRLastf); pCommith->isLFileSame = true; - if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + // if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) { + if (tsdbOpenDFile(pWLastf, TD_FILE_WRITE) < 0) { tsdbError("vgId:%d failed to open file %s to commit since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pWLastf), tstrerror(terrno)); diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index e5d5ef513f275aceaafd4ade9a3c6e79bc90ed17..24c765d3e5939c4dcb3fb9cb6244a25592b6f4bc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -416,8 +416,8 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbGetTxnFname(pRepo, TSDB_TXN_TEMP_FILE, tfname); tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, cfname); - int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(tfname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -436,9 +436,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE); - if (taosWriteFile(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + if (taosWriteFile(pFile, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -446,7 +446,7 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { // Encode file status and write to file if (fsheader.len > 0) { if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) { - close(fd); + taosCloseFile(&pFile); remove(tfname); return -1; } @@ -455,9 +455,9 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { tsdbEncodeFSStatus(&ptr, pStatus); taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len); - if (taosWriteFile(fd, pBuf, fsheader.len) < fsheader.len) { + if (taosWriteFile(pFile, pBuf, fsheader.len) < fsheader.len) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); (void)remove(tfname); taosTZfree(pBuf); return -1; @@ -465,15 +465,15 @@ static int tsdbSaveFSStatus(STsdb *pRepo, SFSStatus *pStatus) { } // fsync, close and rename - if (taosFsyncFile(fd) < 0) { + if (taosFsyncFile(pFile) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); + taosCloseFile(&pFile); remove(tfname); taosTZfree(pBuf); return -1; } - (void)close(fd); + (void)taosCloseFile(&pFile); (void)taosRenameFile(tfname, cfname); taosTZfree(pBuf); @@ -652,7 +652,7 @@ static void tsdbGetTxnFname(STsdb *pRepo, TSDB_TXN_FILE_T ftype, char fname[]) { static int tsdbOpenFSFromCurrent(STsdb *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); - int fd = -1; + TdFilePtr pFile = NULL; void * buffer = NULL; SFSHeader fsheader; char current[TSDB_FILENAME_LEN] = "\0"; @@ -661,8 +661,8 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { tsdbGetTxnFname(pRepo, TSDB_TXN_CURR_FILE, current); // current file exists, try to recover - fd = open(current, O_RDONLY | O_BINARY); - if (fd < 0) { + pFile = taosOpenFile(current, TD_FILE_READ); + if (pFile == NULL) { tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -672,7 +672,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - int nread = (int)taosReadFile(fd, buffer, TSDB_FILE_HEAD_SIZE); + int nread = (int)taosReadFile(pFile, buffer, TSDB_FILE_HEAD_SIZE); if (nread < 0) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current, strerror(errno)); @@ -706,7 +706,7 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { goto _err; } - nread = (int)taosReadFile(fd, buffer, fsheader.len); + nread = (int)taosReadFile(pFile, buffer, fsheader.len); if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -732,13 +732,13 @@ static int tsdbOpenFSFromCurrent(STsdb *pRepo) { } taosTZfree(buffer); - close(fd); + taosCloseFile(&pFile); return 0; _err: - if (fd >= 0) { - close(fd); + if (pFile != NULL) { + taosCloseFile(&pFile); } taosTZfree(buffer); return -1; @@ -1229,7 +1229,8 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { pDFile->f = *pf; - if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + // if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) { + if (tsdbOpenDFile(pDFile, TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); taosArrayDestroy(fArray); @@ -1244,18 +1245,17 @@ static int tsdbRestoreDFileSet(STsdb *pRepo) { } if (tsdbForceKeepFile) { - struct stat tfstat; - + int64_t file_size; // Get real file size - if (fstat(pDFile->fd, &tfstat) < 0) { + if (taosFStatFile(pDFile->pFile, &file_size, NULL) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); taosArrayDestroy(fArray); return -1; } - if (pDFile->info.size != tfstat.st_size) { + if (pDFile->info.size != file_size) { int64_t tfsize = pDFile->info.size; - pDFile->info.size = tfstat.st_size; + pDFile->info.size = file_size; tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); } @@ -1339,7 +1339,8 @@ static void tsdbScanAndTryFixDFilesHeader(STsdb *pRepo, int32_t *nExpired) { } tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); - if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + // if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + if (tsdbOpenDFileSet(&fset, TD_FILE_WRITE | TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 579160bcb2cadafc584ffedc25932eb1ea5c5936..5a00fa20a0d530189a41b048ad1a1b0e515248ef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -355,8 +355,8 @@ static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) { int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile == NULL) { if (errno == ENOENT) { // Try to create directory recursively char *s = strdup(TSDB_FILE_REL_NAME(pDFile)); @@ -366,8 +366,8 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader) { } tfree(s); - pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755); - if (pDFile->fd < 0) { + pDFile->pFile = taosOpenFile(TSDB_FILE_FULL_NAME(pDFile), TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pDFile->pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -456,11 +456,12 @@ static int tsdbScanAndTryFixDFile(STsdb *pRepo, SDFile *pDFile) { } if (pDFile->info.size < dfstat.st_size) { - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(df.fd, df.info.size) < 0) { + if (taosFtruncateFile(df.pFile, df.info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; @@ -537,11 +538,12 @@ static int tsdbApplyDFileChange(SDFile *from, SDFile *to) { static int tsdbRollBackDFile(SDFile *pDFile) { SDFile df = *pDFile; - if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + // if (tsdbOpenDFile(&df, O_WRONLY) < 0) { + if (tsdbOpenDFile(&df, TD_FILE_WRITE) < 0) { return -1; } - if (taosFtruncateFile(TSDB_FILE_FD(&df), pDFile->info.size) < 0) { + if (taosFtruncateFile(TSDB_FILE_PFILE(&df), pDFile->info.size) < 0) { terrno = TAOS_SYSTEM_ERROR(errno); tsdbCloseDFile(&df); return -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index bed2c0cd41d3b2e37d781c98d9853c59a2d89771..c1cf776cb542110f6089ab9e684d2e2756a8040c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -19,7 +19,7 @@ #include "tsdbLog.h" #include "tsdbReadImpl.h" #include "ttime.h" -#include "exception.h" +#include "texception.h" #include "os.h" #include "talgo.h" #include "tcompare.h" @@ -88,7 +88,7 @@ typedef struct STableCheckInfo { int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks uint8_t chosen:2; // indicate which iterator should move forward - bool initBuf; // whether to initialize the in-memory skip list iterator or not + bool initBuf:1; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator } STableCheckInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index d06e37286d457506d1f1198e8f66a60bc37123ae..7e8e7866d71addb8a125998bf3fa54963b2466cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -83,7 +83,8 @@ int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) { pReadh->rSet = *pSet; TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh)); - if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + // if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) { + if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), TD_FILE_READ) < 0) { tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet), tstrerror(terrno)); return -1; diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index a2454e4c5e731c872061eb97a1a076352b7c8a0c..e07e46948f4b66930ccfa3abb7d31066f12cad72 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,3 +1,4 @@ +add_definitions("-D ALLOW_FORBID_FUNC") add_subdirectory(transport) add_subdirectory(sync) add_subdirectory(tdb) @@ -14,4 +15,4 @@ add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) add_subdirectory(nodes) -add_subdirectory(scalar) \ No newline at end of file +add_subdirectory(scalar) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 5857437f9d6381e34620c093c8532cdb847ffb6b..e71f559ad960466264702e67b6c384d01e5f1113 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "catalog.h" -#include "common.h" +#include "tcommon.h" #include "query.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 @@ -189,11 +189,21 @@ typedef struct SCtgAction { #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) -#define CTG_IS_STABLE(isSTable) (1 == (isSTable)) -#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable)) -#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0) -#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0) -#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_FLAG_STB 0x1 +#define CTG_FLAG_NOT_STB 0x2 +#define CTG_FLAG_UNKNOWN_STB 0x4 +#define CTG_FLAG_INF_DB 0x8 + +#define CTG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) +#define CTG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) +#define CTG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) +#define CTG_IS_INF_DB(_flag) ((_flag) & CTG_FLAG_INF_DB) +#define CTG_SET_INF_DB(_flag) ((_flag) |= CTG_FLAG_INF_DB) +#define CTG_SET_STB(_flag, tbType) do { (_flag) |= ((tbType) == TSDB_SUPER_TABLE) ? CTG_FLAG_STB : ((tbType) > TSDB_SUPER_TABLE ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB); } while (0) +#define CTG_GEN_STB_FLAG(_isStb) ((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB) +#define CTG_TBTYPE_MATCH(_flag, tbType) (CTG_IS_UNKNOWN_STB(_flag) || (CTG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) + +#define CTG_IS_INF_DBNAME(_dbname) ((*(_dbname) == 'i') && (0 == strcmp(_dbname, TSDB_INFORMATION_SCHEMA_DB))) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d62c189d33cd7c7621f8c65d3cf8981b9cf754b2..2b6caa2f8fb91f791f2ab30989398e9e12bf754e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -509,7 +509,7 @@ int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, } -int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { +int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist, int32_t flag) { if (NULL == pCtg->dbCache) { *exist = 0; ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); @@ -517,7 +517,11 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } *pTableMeta = NULL; @@ -590,15 +594,19 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType) { +int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_t *tbType, int32_t flag) { if (NULL == pCtg->dbCache) { ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); - + if (CTG_IS_INF_DB(flag)) { + strcpy(dbFName, pTableName->dbname); + } else { + tNameGetFullDbName(pTableName, dbFName); + } + SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -629,7 +637,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const SName* pTableName, int32_ return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; @@ -637,21 +645,21 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); - int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { ctgError("Build mnode stablemeta msg failed, code:%x", code); CTG_ERR_RET(code); } SRpcMsg rpcMsg = { - .msgType = TDMT_MND_STB_META, + .msgType = TDMT_MND_TABLE_META, .pCont = msg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -664,7 +672,7 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S CTG_ERR_RET(rpcRsp.code); } - code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); @@ -675,15 +683,15 @@ int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTransporter, const S return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { +int32_t ctgGetTableMetaFromMnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); + return ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, dbFName, (char *)pTableName->tname, output); } -int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { +int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -709,7 +717,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTransporter, const SEpSe }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp); + rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { @@ -829,7 +837,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { +int32_t ctgStbVersionCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { @@ -1078,6 +1086,10 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db added to cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); + if (CTG_IS_INF_DBNAME(dbFName)) { + return TSDB_CODE_SUCCESS; + } + CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, dbId); @@ -1100,7 +1112,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { uint64_t *suid = NULL; taosHashGetKey(pIter, (void **)&suid, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgSTableVersionCompare)) { + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) { ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); } @@ -1258,7 +1270,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgSTableVersionCompare); + ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare); } } @@ -1428,15 +1440,17 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) -int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable, STableMetaOutput **pOutput) { - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { +int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t flag, STableMetaOutput **pOutput) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; int32_t code = 0; - CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo)); + if (!CTG_IS_INF_DB(flag)) { + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo)); + } SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; @@ -1445,33 +1459,37 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - - if (CTG_IS_STABLE(isSTable)) { + + if (CTG_IS_INF_DB(flag)) { + ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(pTableName)); + + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, (char *)pTableName->dbname, (char *)pTableName->tname, output)); + } else if (CTG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode - CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTransporter, pMgmtEps, pTableName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCtg, pTrans, pMgmtEps, pTableName, output)); if (CTG_IS_META_NULL(output->metaType)) { - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pTableName), flag); // if get from vnode failed or no table meta, will not try mnode - CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTransporter, pMgmtEps, pTableName, &vgroupInfo, output)); + CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCtg, pTrans, pMgmtEps, pTableName, &vgroupInfo, output)); if (CTG_IS_META_TABLE(output->metaType) && TSDB_SUPER_TABLE == output->tbMeta->tableType) { ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), output->metaType); tfree(output->tbMeta); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, output)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, output)); } else if (CTG_IS_META_BOTH(output->metaType)) { int32_t exist = 0; CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTransporter, pMgmtEps, output->dbFName, output->tbName, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(output->metaType); @@ -1530,7 +1548,7 @@ _return: CTG_RET(code); } -int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) { +int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t flag) { if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1538,26 +1556,30 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t exist = 0; int32_t code = 0; - if (!forceUpdate) { - CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist)); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + CTG_SET_INF_DB(flag); + } - if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) { + if ((!forceUpdate) || (CTG_IS_INF_DB(flag))) { + CTG_ERR_RET(ctgGetTableMetaFromCache(pCtg, pTableName, pTableMeta, &exist, flag)); + + if (exist && CTG_TBTYPE_MATCH(flag, (*pTableMeta)->tableType)) { return TSDB_CODE_SUCCESS; } tfree(*pTableMeta); - } else if (CTG_IS_UNKNOWN_STABLE(isSTable)) { + } else if (CTG_IS_UNKNOWN_STB(flag)) { int32_t tbType = 0; - CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType)); + CTG_ERR_RET(ctgGetTableTypeFromCache(pCtg, pTableName, &tbType, flag)); - CTG_SET_STABLE(isSTable, tbType); + CTG_SET_STB(flag, tbType); } STableMetaOutput *output = NULL; while (true) { - CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); + CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, flag, &output)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; @@ -1582,7 +1604,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons SName stbName = *pTableName; strcpy(stbName.tname, output->tbName); - CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); + CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist, flag)); if (0 == exist) { ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); continue; @@ -1663,6 +1685,11 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action) { CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } + char *p = strchr(output->dbFName, '.'); + if (p && CTG_IS_INF_DBNAME(p + 1)) { + memmove(output->dbFName, p + 1, strlen(p + 1)); + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); if (NULL == dbCache) { ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); @@ -1729,7 +1756,7 @@ int32_t ctgActRemoveStb(SCtgMetaAction *action) { ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgSTableVersionCompare)); + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionCompare)); ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2140,16 +2167,16 @@ _return: } -int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_UNKNOWN_STB)); } -int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { +int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, false, pTableMeta, CTG_FLAG_STB)); } int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { @@ -2204,20 +2231,20 @@ _return: } -int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { +int32_t catalogRefreshTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pTableName) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTransporter, pMgmtEps, pTableName, isSTable, NULL)); + CTG_API_LEAVE(ctgRefreshTblMeta(pCtg, pTrans, pMgmtEps, pTableName, CTG_GEN_STB_FLAG(isSTable), NULL)); } -int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { +int32_t catalogRefreshGetTableMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) { CTG_API_ENTER(); - CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable)); + CTG_API_LEAVE(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, pTableName, true, pTableMeta, CTG_GEN_STB_FLAG(isSTable))); } int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgList) { @@ -2226,6 +2253,11 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pVgList) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } STableMeta *tbMeta = NULL; int32_t code = 0; @@ -2236,7 +2268,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm *pVgList = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pRpc, pMgmtEps, pTableName, false, &tbMeta, CTG_FLAG_UNKNOWN_STB)); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -2307,16 +2339,21 @@ _return: } -int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { +int32_t catalogGetTableHashVgroup(SCatalog *pCtg, void *pTrans, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { CTG_API_ENTER(); + if (CTG_IS_INF_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); SDBVgInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTransporter, pMgmtEps, db, false, &dbCache, &vgInfo)); + CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pTrans, pMgmtEps, db, false, &dbCache, &vgInfo)); CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); @@ -2336,10 +2373,10 @@ _return: } -int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { +int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pTransporter || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { + if (NULL == pCtg || NULL == pTrans || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } @@ -2363,7 +2400,7 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm SName *name = taosArrayGet(pReq->pTableName, i); STableMeta *pTableMeta = NULL; - CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTransporter, pMgmtEps, name, false, &pTableMeta, -1)); + CTG_ERR_JRET(ctgGetTableMeta(pCtg, pTrans, pMgmtEps, name, false, &pTableMeta, CTG_FLAG_UNKNOWN_STB)); if (NULL == taosArrayPush(pRsp->pTableMeta, &pTableMeta)) { ctgError("taosArrayPush failed, idx:%d", i); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 1c84f06a2243b56f7e3427864bd3fc502dcc5343..f3aecadd3fef9fb3231ef720e23954058861ca67 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -38,7 +38,7 @@ namespace { extern "C" int32_t ctgGetTableMetaFromCache(struct SCatalog *pCatalog, const SName *pTableName, STableMeta **pTableMeta, - int32_t *exist); + int32_t *exist, int32_t flag); extern "C" int32_t ctgDbgGetClusterCacheNum(struct SCatalog* pCatalog, int32_t type); extern "C" int32_t ctgActUpdateTbl(SCtgMetaAction *action); extern "C" int32_t ctgDbgEnableDebug(char *option); @@ -131,9 +131,7 @@ void ctgTestInitLogFile() { ctgDbgEnableDebug("api"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } @@ -245,6 +243,8 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->suid = ctgTestSuid + 1; rspMsg->tuid = ctgTestSuid + 1; rspMsg->vgId = 1; + + rspMsg->pSchemas = (SSchema *)calloc(rspMsg->numOfTags + rspMsg->numOfColumns, sizeof(SSchema)); SSchema *s = NULL; s = &rspMsg->pSchemas[0]; @@ -772,7 +772,7 @@ void *ctgTestGetCtableMetaThread(void *param) { strcpy(cn.tname, ctgTestCTablename); while (!ctgTestStop) { - code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist); + code = ctgGetTableMetaFromCache(pCtg, &cn, &tbMeta, &exist, 0); if (code || 0 == exist) { assert(0); } @@ -829,6 +829,7 @@ void *ctgTestSetCtableMetaThread(void *param) { #if 0 + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1291,6 +1292,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogUpdateSTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); + tfree(rsp.pSchemas); while (true) { uint64_t n = 0; diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 700356436556bc7f1bb5e892f5487848b6a70371..f15be8521c1298e75cbf8b66304a715a2f99dac0 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "dataSinkMgt.h" struct SDataSink; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 10d884cb3f9e1350c199832f5b8021a6b923f71e..cff7546a7eff2fc7895ac0e596ca1384e8d4ec2b 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,7 +15,7 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H -#include "common.h" +#include "tcommon.h" #include "tbuffer.h" #include "tpagedbuf.h" @@ -68,9 +68,10 @@ typedef struct SResultRow { } SResultRow; typedef struct SResultRowInfo { + SResultRow *pCurResult; // current active result row info SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set +// int16_t type:8; // data type for hash key + int32_t size; // number of result set int32_t capacity; // max capacity int32_t curPos; // current active result row index of pResult list } SResultRowInfo; @@ -95,7 +96,7 @@ struct SUdfInfo; int32_t getOutputInterResultBufSize(struct STaskAttr* pQueryAttr); size_t getResultRowSize(SArray* pExprInfo); -int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size, int16_t type); +int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); void resetResultRowInfo(struct STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo); @@ -105,7 +106,7 @@ void closeAllResultRows(SResultRowInfo* pResultRowInfo); int32_t initResultRow(SResultRow *pResultRow); void closeResultRow(SResultRowInfo* pResultRowInfo, int32_t slot); bool isResultRowClosed(SResultRowInfo *pResultRowInfo, int32_t slot); -void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow, int16_t type); +void clearResultRow(struct STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, int32_t* offset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 23acdf1c336adc1324113d4ffc3582f34a02cf4a..c8e1aeba0eff7cd9ff9bdf16df16fba3360160ed 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -15,12 +15,13 @@ #ifndef TDENGINE_EXECUTORIMPL_H #define TDENGINE_EXECUTORIMPL_H +#include "tsort.h" #ifdef __cplusplus extern "C" { #endif #include "os.h" -#include "common.h" +#include "tcommon.h" #include "tlosertree.h" #include "ttszip.h" #include "tvariant.h" @@ -444,16 +445,32 @@ typedef struct SOptrBasicInfo { int32_t capacity; } SOptrBasicInfo; -typedef struct SOptrBasicInfo STableIntervalOperatorInfo; - -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file +typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. +} SAggSupporter; + +typedef struct STableIntervalOperatorInfo { + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SGroupResInfo groupResInfo; + SInterval interval; + STimeWindow win; + int32_t precision; + bool timeWindowInterpo; + char **pRow; + SAggSupporter aggSup; + STableQueryInfo *pCurrent; + int32_t order; +} STableIntervalOperatorInfo; + +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SAggSupporter aggSup; STableQueryInfo *current; uint32_t groupId; SGroupResInfo groupResInfo; @@ -549,49 +566,42 @@ typedef struct SDistinctOperatorInfo { SArray* pDistinctDataInfo; } SDistinctOperatorInfo; -struct SGlobalMerger; - -typedef struct SMultiwayMergeInfo { - struct SGlobalMerger* pMerge; - SOptrBasicInfo binfo; - int32_t bufCapacity; - int64_t seed; - char** prevRow; - SArray* orderColumnList; - int32_t resultRowFactor; - - bool hasGroupColData; - char** currentGroupColData; - SArray* groupColumnList; - bool hasDataBlockForNewGroup; - SSDataBlock* pExistBlock; - - SArray* udfInfo; - bool hasPrev; - bool multiGroupResults; -} SMultiwayMergeInfo; - -typedef struct SMsortComparParam { - struct SExternalMemSource **pSources; - int32_t numOfSources; - SArray *orderInfo; // SArray - bool nullFirst; -} SMsortComparParam; +typedef struct SSortedMergeOperatorInfo { + SOptrBasicInfo binfo; + bool hasVarCol; + + SArray *orderInfo; // SArray + bool nullFirst; + int32_t numOfSources; + + SSortHandle *pSortHandle; + + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + + int32_t resultRowFactor; + bool hasGroupVal; + + SDiskbasedBuf *pTupleStore; // keep the final results + int32_t numOfResPerPage; + + char** groupVal; + SArray *groupInfo; + SAggSupporter aggSup; +} SSortedMergeOperatorInfo; typedef struct SOrderOperatorInfo { - int32_t sourceId; uint32_t sortBufSize; // max buffer size for in-memory sort SSDataBlock *pDataBlock; bool hasVarCol; // has variable length column, such as binary/varchar/nchar - int32_t numOfCompleted; - SDiskbasedBuf *pSortInternalBuf; - SMultiwayMergeTreeInfo *pMergeTree; - SArray *pSources; // SArray + SArray *orderInfo; + bool nullFirst; + SSortHandle *pSortHandle; + int32_t bufPageSize; int32_t numOfRowsInRes; - SMsortComparParam cmpParam; - + // TODO extact struct int64_t startTs; // sort start time uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. uint64_t totalSize; // total load bytes from remote @@ -608,8 +618,8 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, - int32_t numOfOutput); +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); + SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, @@ -641,9 +651,8 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal); -SOperatorInfo* createMergeSortOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, - SOrder* pOrderVal); +SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); // SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); // SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); @@ -691,9 +700,6 @@ int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status); -bool onlyQueryTags(STaskAttr* pQueryAttr); -// void destroyUdfInfo(struct SUdfInfo* pUdfInfo); - int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen); size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows); diff --git a/include/common/tcompare.h b/source/libs/executor/inc/tlinearhash.h similarity index 50% rename from include/common/tcompare.h rename to source/libs/executor/inc/tlinearhash.h index 5dcbf37b90c03ce0a669d214544dd01da95286da..a4190582166d565d42faf062c78b87c65ccdf644 100644 --- a/include/common/tcompare.h +++ b/source/libs/executor/inc/tlinearhash.h @@ -13,28 +13,32 @@ * along with this program. If not, see . */ -#ifndef _TD_TCOMPARE_H_ -#define _TD_TCOMPARE_H_ - -#include "compare.h" -#include "ttypes.h" +#ifndef TDENGINE_TLINEARHASH_H +#define TDENGINE_TLINEARHASH_H #ifdef __cplusplus extern "C" { #endif -int32_t compareStrPatternMatch(const void* pLeft, const void* pRight); -int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight); +#include "thash.h" + +enum { + LINEAR_HASH_STATIS = 0x1, + LINEAR_HASH_DATA = 0x2, +}; + +typedef struct SLHashObj SLHashObj; -int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight); -int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight); +SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage); +void* tHashCleanup(SLHashObj* pHashObj); -__compar_fn_t getComparFunc(int32_t type, int32_t optr); -__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order); -int32_t doCompare(const char* a, const char* b, int32_t type, size_t size); +int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size); +char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen); +int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen); + +void tHashPrint(const SLHashObj* pHashObj, int32_t type); #ifdef __cplusplus } #endif - -#endif /*_TD_TCOMPARE_H_*/ +#endif // TDENGINE_TLINEARHASH_H diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h new file mode 100644 index 0000000000000000000000000000000000000000..ef7af2b4e3e08d884a5e7cc16118603746e0de02 --- /dev/null +++ b/source/libs/executor/inc/tsort.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TSORT_H +#define TDENGINE_TSORT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tcommon.h" +#include "os.h" + +enum { + SORT_MULTISOURCE_MERGE = 0x1, + SORT_SINGLESOURCE_SORT = 0x2, +}; + +typedef struct SMultiMergeSource { + int32_t type; + int32_t rowIndex; + SSDataBlock *pBlock; +} SMultiMergeSource; + +typedef struct SExternalMemSource { + SMultiMergeSource src; + SArray* pageIdList; + int32_t pageIndex; +} SExternalMemSource; + +typedef struct SGenericSource { + SMultiMergeSource src; + void *param; +} SGenericSource; + +typedef struct SMsortComparParam { + void **pSources; + int32_t numOfSources; + SArray *orderInfo; // SArray + bool nullFirst; +} SMsortComparParam; + +typedef struct SSortHandle SSortHandle; +typedef struct STupleHandle STupleHandle; + +typedef SSDataBlock* (*_sort_fetch_block_fn_t)(void* param); +typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void* param); + +/** + * + * @param type + * @return + */ +SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr); + +/** + * + * @param pSortHandle + */ +void tsortDestroySortHandle(SSortHandle* pSortHandle); + +/** + * + * @param pHandle + * @return + */ +int32_t tsortOpen(SSortHandle* pHandle); + +/** + * + * @param pHandle + * @return + */ +int32_t tsortClose(SSortHandle* pHandle); + +/** + * + * @return + */ +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp); + +/** + * + * @param pHandle + * @param fp + * @return + */ +int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); + +/** + * + * @param pHandle + * @param pSource + * @return success or failed + */ +int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource); + +/** + * + * @param pHandle + * @return + */ +STupleHandle* tsortNextTuple(SSortHandle* pHandle); + +/** + * + * @param pHandle + * @param colIndex + * @return + */ +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex); + +/** + * + * @param pHandle + * @param colIndex + * @return + */ +void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TSORT_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 52ab8493f1841f511a9f00b41be438899a60666f..1b901ee9f684e1894ad66128c47df6ca732126f4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -53,8 +53,7 @@ int32_t getOutputInterResultBufSize(STaskAttr* pQueryAttr) { return size; } -int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { - pResultRowInfo->type = type; +int32_t initResultRowInfo(SResultRowInfo *pResultRowInfo, int32_t size) { pResultRowInfo->size = 0; pResultRowInfo->curPos = -1; pResultRowInfo->capacity = size; @@ -93,7 +92,7 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRow *pWindowRes = pResultRowInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); + clearResultRow(pRuntimeEnv, pWindowRes); int32_t groupIndex = 0; int64_t uid = 0; @@ -136,7 +135,7 @@ void closeResultRow(SResultRowInfo *pResultRowInfo, int32_t slot) { getResultRow(pResultRowInfo, slot)->closed = true; } -void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_t type) { +void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow) { if (pResultRow == NULL) { return; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4078ee9291963724ee69989021552c089239bf01..b90ce275d21ae3645d4ea5cb966b6eed215ae716 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -93,3 +93,25 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return pTaskInfo; } + +int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isAdd) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo* )tinfo; + + // traverse to the streamscan node to add this table id + SOperatorInfo* pInfo = pTaskInfo->pRoot; + while(pInfo->operatorType != OP_StreamScan) { + pInfo = pInfo->pDownstream[0]; + } + + SStreamBlockScanInfo* pScanInfo = pInfo->info; + if (isAdd) { + int32_t code = tqReadHandleSetTbUidList(pScanInfo->readerHandle, tableIdList); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + assert(0); + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index f632a84c1f3d7c737d60fdaa4149eaf9ea896a26..1642120d150148da8b5ac5455accebd934810ddd 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -15,7 +15,7 @@ #include #include "dataSinkMgt.h" -#include "exception.h" +#include "texception.h" #include "os.h" #include "tarray.h" #include "tcache.h" diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5508b5ecd713016a4d228a7087d34faaadddc38f..c400ded1c1889d7781fc74b4506c873110e56401 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,13 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "exception.h" + #include "os.h" + +#include "tep.h" +#include "tsort.h" +#include "texception.h" #include "parser.h" #include "tglobal.h" #include "tmsg.h" -#include "tq.h" #include "ttime.h" #include "executorimpl.h" @@ -140,19 +142,19 @@ static int32_t getExprFunctionId(SExprInfo *pExprInfo) { return 0; } -static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - if (pQueryAttr->interval.intervalUnit != 'n' && pQueryAttr->interval.intervalUnit != 'y') { - tw->skey += pQueryAttr->interval.sliding * factor; - tw->ekey = tw->skey + pQueryAttr->interval.interval - 1; +static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t order, STimeWindow* tw) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); + if (pInterval->intervalUnit != 'n' && pInterval->intervalUnit != 'y') { + tw->skey += pInterval->sliding * factor; + tw->ekey = tw->skey + pInterval->interval - 1; return; } - int64_t key = tw->skey, interval = pQueryAttr->interval.interval; + int64_t key = tw->skey, interval = pInterval->interval; //convert key to second - key = convertTimePrecision(key, pQueryAttr->precision, TSDB_TIME_PRECISION_MILLI) / 1000; + key = convertTimePrecision(key, precision, TSDB_TIME_PRECISION_MILLI) / 1000; - if (pQueryAttr->interval.intervalUnit == 'y') { + if (pInterval->intervalUnit == 'y') { interval *= 12; } @@ -163,12 +165,12 @@ static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->skey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pQueryAttr->precision); + tw->ekey = convertTimePrecision((int64_t)mktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); tw->ekey -= 1; } @@ -178,7 +180,7 @@ static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult int32_t numOfCols, int32_t* rowCellInfoOffset); void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx); +static bool functionNeedToExecute(SqlFunctionCtx *pCtx); static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); @@ -221,7 +223,8 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); -static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); +static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); + static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); @@ -380,18 +383,23 @@ static bool hasNull(SColumn* pColumn, SColumnDataAgg *pStatis) { } static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { + int64_t newCapacity = 0; + // more than the capacity, reallocate the resources if (pResultRowInfo->size < pResultRowInfo->capacity) { return; } - int64_t newCapacity = 0; if (pResultRowInfo->capacity > 10000) { newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); } else { newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); } + if (newCapacity == pResultRowInfo->capacity) { + newCapacity += 4; + } + char *t = realloc(pResultRowInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); if (t == NULL) { longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -518,12 +526,12 @@ static SResultRow* doSetResultOutBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultR } static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t tid, char* pData, int16_t bytes, - bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggOperatorInfo* pAggInfo) { + bool masterscan, uint64_t tableGroupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { bool existed = false; - SET_RES_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tableGroupId); + SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, tableGroupId); SResultRow **p1 = - (SResultRow **)taosHashGet(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + (SResultRow **)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); // in case of repeat scan/reverse scan, no new time window added. if (isIntervalQuery) { @@ -539,8 +547,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int existed = (pResultRowInfo->pResult[0] == (*p1)); pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); + SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); + int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; existed = true; @@ -561,16 +569,16 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int SResultRow *pResult = NULL; if (p1 == NULL) { - pResult = getNewResultRow(pAggInfo->pool); + pResult = getNewResultRow(pSup->pool); int32_t ret = initResultRow(pResult); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // add a new result set for a new group - taosHashPut(pAggInfo->pResultRowHashTable, pAggInfo->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES); SResultRowCell cell = {.groupId = tableGroupId, .pRow = pResult}; - taosArrayPush(pAggInfo->pResultRowArrayList, &cell); + taosArrayPush(pSup->pResultRowArrayList, &cell); } else { pResult = *p1; } @@ -579,8 +587,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; int64_t index = pResultRowInfo->curPos; - SET_RES_EXT_WINDOW_KEY(pAggInfo->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pAggInfo->pResultRowListSet, pAggInfo->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, tid, pResultRowInfo); + taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); } // too many time window in query @@ -591,19 +599,19 @@ static SResultRow* doSetResultOutBufByKey_rv(SResultRowInfo* pResultRowInfo, int return pResultRowInfo->pResult[pResultRowInfo->curPos]; } -static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWindow* w) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, ts, ts, pQueryAttr->window.ekey, w); +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, TSKEY ekey, bool ascQuery) { + if (ascQuery) { + getAlignQueryTimeWindow(pInterval, precision, ts, ts, ekey, w); } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp - getAlignQueryTimeWindow(pQueryAttr, ts, pQueryAttr->window.ekey, ts, w); + getAlignQueryTimeWindow(pInterval, precision, ts, ekey, ts, w); int64_t key = w->skey; while(key < ts) { // moving towards end - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - key = taosTimeAdd(key, pQueryAttr->interval.sliding, pQueryAttr->interval.slidingUnit, pQueryAttr->precision); + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + key = taosTimeAdd(key, pInterval->sliding, pInterval->slidingUnit, precision); } else { - key += pQueryAttr->interval.sliding; + key += pInterval->sliding; } if (key >= ts) { @@ -616,39 +624,39 @@ static void getInitialStartTimeWindow(STaskAttr* pQueryAttr, TSKEY ts, STimeWind } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, STaskAttr *pQueryAttr) { +static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win) { STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); + getInitialStartTimeWindow(pInterval, precision, ts, &w, win->ekey, true); - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - w.ekey = w.skey + pQueryAttr->interval.interval - 1; + w.ekey = w.skey + pInterval->interval - 1; } } else { w = getResultRow(pResultRowInfo, pResultRowInfo->curPos)->win; } if (w.skey > ts || w.ekey < ts) { - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - w.skey = taosTimeTruncate(ts, &pQueryAttr->interval, pQueryAttr->precision); - w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + w.skey = taosTimeTruncate(ts, pInterval, precision); + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { int64_t st = w.skey; if (st > ts) { - st -= ((st - ts + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + st -= ((st - ts + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; } - int64_t et = st + pQueryAttr->interval.interval - 1; + int64_t et = st + pInterval->interval - 1; if (et < ts) { - st += ((ts - et + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; + st += ((ts - et + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; } w.skey = st; - w.ekey = w.skey + pQueryAttr->interval.interval - 1; + w.ekey = w.skey + pInterval->interval - 1; } } @@ -656,10 +664,7 @@ static STimeWindow getActiveTimeWindow(SResultRowInfo * pResultRowInfo, int64_t * query border check, skey should not be bounded by the query time range, since the value skey will * be used as the time window index value. So we only change ekey of time window accordingly. */ - if (w.ekey > pQueryAttr->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) { - w.ekey = pQueryAttr->window.ekey; - } - +// ASSERT(win->skey <= win->ekey); // todo no need this return w; } @@ -668,7 +673,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, i STimeWindow w = {0}; if (pResultRowInfo->curPos == -1) { // the first window, from the previous stored value - getInitialStartTimeWindow(pQueryAttr, ts, &w); +// getInitialStartTimeWindow(pQueryAttr, ts, &w); if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { w.ekey = taosTimeAdd(w.skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; @@ -703,7 +708,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes SIDList list = getDataBufPagesIdList(pResultBuf, tid); if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, tid, &pageId); + pData = getNewBufPage(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); pData = getBufPage(pResultBuf, getPageId(pi)); @@ -712,7 +717,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pRes if (pData->num + size > getBufPageSize(pResultBuf)) { // release current page first, and prepare the next one releaseBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, tid, &pageId); + pData = getNewBufPage(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer } @@ -770,6 +775,37 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowI return TSDB_CODE_SUCCESS; } +static void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); + +static int32_t setResultOutputBufByKey_rv(SResultRowInfo *pResultRowInfo, int64_t id, STimeWindow *win, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowCellInfoOffset, SDiskbasedBuf *pBuf, SAggSupporter *pAggSup, SExecTaskInfo* pTaskInfo) { + assert(win->skey <= win->ekey); + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pResultRowInfo, id, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, + pTaskInfo, true, pAggSup); + + if (pResultRow == NULL) { + *pResult = NULL; + return TSDB_CODE_SUCCESS; + } + + // not assign result buffer yet, add new result buffer + if (pResultRow->pageId == -1) { // todo intermediate result size + int32_t ret = addNewWindowResultBuf(pResultRow, pBuf, (int32_t) tableGroupId, 0); + if (ret != TSDB_CODE_SUCCESS) { + return -1; + } + } + + // set time window for current result + pResultRow->win = (*win); + *pResult = pResultRow; + setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + + return TSDB_CODE_SUCCESS; +} + + static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { @@ -871,48 +907,44 @@ static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, } } -static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, STaskAttr* pQueryAttr, TSKEY lastKey) { - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); - if ((lastKey > pQueryAttr->window.ekey && ascQuery) || (lastKey < pQueryAttr->window.ekey && (!ascQuery))) { +static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, bool ascQuery, bool interp) { + if ((lastKey > pWin->ekey && ascQuery) || (lastKey < pWin->ekey && (!ascQuery))) { closeAllResultRows(pResultRowInfo); pResultRowInfo->curPos = pResultRowInfo->size - 1; } else { int32_t step = ascQuery ? 1 : -1; - doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, pQueryAttr->timeWindowInterpo); + doUpdateResultRowIndex(pResultRowInfo, lastKey - step, ascQuery, interp); } } -static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, - int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) { +static int32_t getNumOfRowsInTimeWindow(SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn, + int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { assert(startPos >= 0 && startPos < pDataBlockInfo->rows); - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - STableQueryInfo* item = pRuntimeEnv->current; int32_t num = -1; - int32_t order = pQueryAttr->order.order; int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (order == TSDB_ORDER_ASC) { if (ekey < pDataBlockInfo->window.ekey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key + if (item != NULL) { item->lastKey = pPrimaryColumn[startPos + (num - 1)] + step; } } else { num = pDataBlockInfo->rows - startPos; - if (updateLastKey) { + if (item != NULL) { item->lastKey = pDataBlockInfo->window.ekey + step; } } } else { // desc if (ekey > pDataBlockInfo->window.skey && pPrimaryColumn) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); - if (updateLastKey) { // update the last key + if (item != NULL) { item->lastKey = pPrimaryColumn[startPos - (num - 1)] + step; } } else { num = startPos + 1; - if (updateLastKey) { + if (item != NULL) { item->lastKey = pDataBlockInfo->window.skey + step; } } @@ -922,22 +954,18 @@ static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlock return num; } -static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - bool hasAggregates = pCtx[0].isAggSet; - +static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].size = forwardStep; pCtx[k].startTs = pWin->skey; // keep it temporarialy - char* start = pCtx[k].pInput; + int32_t startOffset = pCtx[k].startRow; + bool hasAgg = pCtx[k].isAggSet; - int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1); - if (pCtx[k].pInput != NULL) { - pCtx[k].pInput = (char *)pCtx[k].pInput + pos * pCtx[k].inputBytes; - } + int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); + pCtx[k].startRow = pos; if (tsCol != NULL) { pCtx[k].ptsList = &tsCol[pos]; @@ -949,88 +977,80 @@ static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, pCtx[k].isAggSet = false; } - if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) { + if (functionNeedToExecute(&pCtx[k])) { pCtx[k].fpSet->addInput(&pCtx[k]); } // restore it - pCtx[k].isAggSet = hasAggregates; - pCtx[k].pInput = start; + pCtx[k].isAggSet = hasAgg; + pCtx[k].startRow = startOffset; } } -static int32_t getNextQualifiedWindow(STaskAttr* pQueryAttr, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, - TSKEY* primaryKeys, __block_search_fn_t searchFn, int32_t prevPosition) { - getNextTimeWindow(pQueryAttr, pNext); +static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo, + TSKEY* primaryKeys, int32_t prevPosition, STableIntervalOperatorInfo* pInfo) { + int32_t order = pInfo->order; + bool ascQuery = (order == TSDB_ORDER_ASC); + + int32_t precision = pInfo->precision; + getNextTimeWindow(pInterval, precision, order, pNext); // next time window is not in current block - if ((pNext->skey > pDataBlockInfo->window.ekey && QUERY_IS_ASC_QUERY(pQueryAttr)) || - (pNext->ekey < pDataBlockInfo->window.skey && !QUERY_IS_ASC_QUERY(pQueryAttr))) { + if ((pNext->skey > pDataBlockInfo->window.ekey && order == TSDB_ORDER_ASC) || + (pNext->ekey < pDataBlockInfo->window.skey && order == TSDB_ORDER_DESC)) { return -1; } - TSKEY startKey = -1; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - startKey = pNext->skey; - if (startKey < pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } else { - startKey = pNext->ekey; - if (startKey > pQueryAttr->window.skey) { - startKey = pQueryAttr->window.skey; - } - } - + TSKEY startKey = ascQuery? pNext->skey:pNext->ekey; int32_t startPos = 0; // tumbling time window query, a special case of sliding time window query - if (pQueryAttr->interval.sliding == pQueryAttr->interval.interval && prevPosition != -1) { - int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + if (pInterval->sliding == pInterval->interval && prevPosition != -1) { + int32_t factor = GET_FORWARD_DIRECTION_FACTOR(order); startPos = prevPosition + factor; } else { - if (startKey <= pDataBlockInfo->window.skey && QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (startKey <= pDataBlockInfo->window.skey && ascQuery) { startPos = 0; - } else if (startKey >= pDataBlockInfo->window.ekey && !QUERY_IS_ASC_QUERY(pQueryAttr)) { + } else if (startKey >= pDataBlockInfo->window.ekey && !ascQuery) { startPos = pDataBlockInfo->rows - 1; } else { - startPos = searchFn((char *)primaryKeys, pDataBlockInfo->rows, startKey, pQueryAttr->order.order); + startPos = binarySearchForKey((char *)primaryKeys, pDataBlockInfo->rows, startKey, order); } } /* interp query with fill should not skip time window */ - if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { - return startPos; - } +// if (pQueryAttr->pointInterpQuery && pQueryAttr->fillType != TSDB_FILL_NONE) { +// return startPos; +// } /* * This time window does not cover any data, try next time window, * this case may happen when the time window is too small */ if (primaryKeys == NULL) { - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (ascQuery) { assert(pDataBlockInfo->window.skey <= pNext->ekey); } else { assert(pDataBlockInfo->window.ekey >= pNext->skey); } } else { - if (QUERY_IS_ASC_QUERY(pQueryAttr) && primaryKeys[startPos] > pNext->ekey) { + if (ascQuery && primaryKeys[startPos] > pNext->ekey) { TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - pNext->ekey += ((next - pNext->ekey + pQueryAttr->interval.sliding - 1)/pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->skey = pNext->ekey - pQueryAttr->interval.interval + 1; + pNext->ekey += ((next - pNext->ekey + pInterval->sliding - 1)/pInterval->sliding) * pInterval->sliding; + pNext->skey = pNext->ekey - pInterval->interval + 1; } - } else if ((!QUERY_IS_ASC_QUERY(pQueryAttr)) && primaryKeys[startPos] < pNext->skey) { + } else if ((!ascQuery) && primaryKeys[startPos] < pNext->skey) { TSKEY next = primaryKeys[startPos]; - if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - pNext->skey = taosTimeTruncate(next, &pQueryAttr->interval, pQueryAttr->precision); - pNext->ekey = taosTimeAdd(pNext->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + pNext->skey = taosTimeTruncate(next, pInterval, precision); + pNext->ekey = taosTimeAdd(pNext->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - pNext->skey -= ((pNext->skey - next + pQueryAttr->interval.sliding - 1) / pQueryAttr->interval.sliding) * pQueryAttr->interval.sliding; - pNext->ekey = pNext->skey + pQueryAttr->interval.interval - 1; + pNext->skey -= ((pNext->skey - next + pInterval->sliding - 1) / pInterval->sliding) * pInterval->sliding; + pNext->ekey = pNext->skey + pInterval->interval - 1; } } } @@ -1067,23 +1087,19 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static void saveDataBlockLastRow(STaskRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock, - int32_t rowIndex) { +static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { if (pDataBlock == NULL) { return; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - for (int32_t k = 0; k < pQueryAttr->numOfCols; ++k) { + for (int32_t k = 0; k < numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } -static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* tsCols, int32_t rows) { +static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, bool ascQuery) { TSKEY ts = TSKEY_INITIAL_VAL; - - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); if (tsCols == NULL) { ts = ascQuery? win->skey : win->ekey; } else { @@ -1151,7 +1167,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, pCtx[i].columnIndex); // in case of the block distribution query, the inputBytes is not a constant value. - pCtx[i].pInput = p->pData; + pCtx[i].pInput = p; assert(p->info.colId == pCol->info.colId); if (pCtx[i].functionId < 0) { @@ -1163,14 +1179,14 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, // uint32_t status = aAggs[pCtx[i].functionId].status; // if ((status & (FUNCSTATE_SELECTIVITY | FUNCSTATE_NEED_TS)) != 0) { - SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); +// SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0); // In case of the top/bottom query again the nest query result, which has no timestamp column // don't set the ptsList attribute. - if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - pCtx[i].ptsList = (int64_t*) tsInfo->pData; - } else { - pCtx[i].ptsList = NULL; - } +// if (tsInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP) { +// pCtx[i].ptsList = (int64_t*) tsInfo->pData; +// } else { +// pCtx[i].ptsList = NULL; +// } // } // } else if (TSDB_COL_IS_UD_COL(pCol->flag) && (pOperator->pRuntimeEnv->scanFlag == MERGE_STAGE)) { // SColIndex* pColIndex = &pOperator->pExpr[i].base.colInfo; @@ -1189,7 +1205,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { - if (functionNeedToExecute(NULL, &pCtx[k])) { + if (functionNeedToExecute(&pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct pCtx[k].fpSet->addInput(&pCtx[k]); } @@ -1348,20 +1364,19 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun } static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - if (!pQueryAttr->timeWindowInterpo) { + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, int32_t order, bool timeWindowInterpo) { + if (!timeWindowInterpo) { return; } assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); if (pBlock->pDataBlock == NULL){ // tscError("pBlock->pDataBlock == NULL"); return; } + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, 0); TSKEY *tsCols = (TSKEY *)(pColInfo->pData); @@ -1374,38 +1389,37 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); } // point interpolation does not require the end key time window interpolation. - if (pQueryAttr->pointInterpQuery) { - return; - } +// if (pointInterpQuery) { +// return; +// } // interpolation query does not generate the time window end interpolation done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); if (!done) { int32_t endRowIndex = startPos + (forwardStep - 1) * step; - TSKEY endKey = QUERY_IS_ASC_QUERY(pQueryAttr)? pBlock->info.window.ekey:pBlock->info.window.skey; + TSKEY endKey = (order == TSDB_ORDER_ASC)? pBlock->info.window.ekey:pBlock->info.window.skey; bool interp = setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pQueryAttr->numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_END_INTERP); } } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*) pOperatorInfo->info; - STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; - int32_t numOfOutput = pOperatorInfo->numOfOutput; - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + int32_t numOfOutput = pOperatorInfo->numOfOutput; - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); - bool ascQuery = QUERY_IS_ASC_QUERY(pQueryAttr); + int32_t step = 1; + bool ascQuery = true; int32_t prevIndex = pResultRowInfo->curPos; @@ -1418,26 +1432,26 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); - STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); - bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); + STimeWindow win = getActiveTimeWindow(pResultRowInfo, ts, &pInfo->interval, pInfo->precision, &pInfo->win); + bool masterScan = true; SResultRow* pResult = NULL; - int32_t ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); + int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } int32_t forwardStep = 0; - TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); + TSKEY ekey = win.ekey; forwardStep = - getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // prev time window not interpolation yet. int32_t curIndex = pResultRowInfo->curPos; - if (prevIndex != -1 && prevIndex < curIndex && pQueryAttr->timeWindowInterpo) { + if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. SResultRow* pRes = getResultRow(pResultRowInfo, j); if (pRes->closed) { @@ -1446,64 +1460,64 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul } STimeWindow w = pRes->win; - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &w, masterScan, &pResult, + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, pInfo, pSDataBlock->pDataBlock, *(TSKEY*)pRuntimeEnv->prevRow[0], -1, + doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pSDataBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->pCtx, pQueryAttr->numOfOutput, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfOutput, RESULT_ROW_START_INTERP); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput); + doApplyFunctions(pInfo->binfo.pCtx, &w, startPos, 0, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } // restore current time window - ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->pCtx, - numOfOutput, pInfo->rowCellInfoOffset); + ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); + doApplyFunctions(pInfo->binfo.pCtx, &win, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; while (1) { int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &nextWin, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo); if (startPos < 0) { break; } // null data, failed to allocate more memory buffer - int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + int32_t code = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &nextWin, masterScan, &pResult, tableGroupId, + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, pInfo->pResultBuf, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - ekey = reviseWindowEkey(pQueryAttr, &nextWin); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); + ekey = nextWin.ekey;//reviseWindowEkey(pQueryAttr, &nextWin); + forwardStep = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &nextWin, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, pInfo->order, false); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } - if (pQueryAttr->timeWindowInterpo) { + if (pInfo->timeWindowInterpo) { int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); + saveDataBlockLastRow(pInfo->pRow, pSDataBlock->pDataBlock, rowIndex, pSDataBlock->info.numOfCols); } - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +// updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); } @@ -1526,7 +1540,7 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } int32_t startPos = ascQuery? 0 : (pSDataBlock->info.rows - 1); - TSKEY ts = getStartTsKey(pQueryAttr, &pSDataBlock->info.window, tsCols, pSDataBlock->info.rows); + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascQuery); STimeWindow win = getCurrentActiveTimeWindow(pResultRowInfo, ts, pQueryAttr); bool masterScan = IS_MAIN_SCAN(pRuntimeEnv); @@ -1539,25 +1553,25 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe while (1) { // null data, failed to allocate more memory buffer ret = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); if (ret != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } TSKEY ekey = reviseWindowEkey(pQueryAttr, &win); - forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); +// forwardStep = getNumOfRowsInTimeWindow(pRuntimeEnv, &pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, true); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); +// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); preWin = win; int32_t prevEndPos = (forwardStep - 1) * step + startPos; - startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); +// startPos = getNextQualifiedWindow(pQueryAttr, &win, &pSDataBlock->info, tsCols, binarySearchForKey, prevEndPos); if (startPos < 0) { if ((ascQuery && win.skey <= pQueryAttr->window.ekey) || ((!ascQuery) && win.ekey >= pQueryAttr->window.ekey)) { int32_t code = setResultOutputBufByKey(pRuntimeEnv, pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->pCtx, numOfOutput, pInfo->rowCellInfoOffset); + pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -1565,8 +1579,8 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe startPos = pSDataBlock->info.rows - 1; // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->pCtx, pResult, &win, startPos, forwardStep); - doApplyFunctions(pRuntimeEnv, pInfo->pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); +// doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, ascQuery ? &win : &preWin, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput); } break; @@ -1576,10 +1590,10 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pQueryAttr->timeWindowInterpo) { int32_t rowIndex = ascQuery? (pSDataBlock->info.rows-1):0; - saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); +// saveDataBlockLastRow(pRuntimeEnv, &pSDataBlock->info, pSDataBlock->pDataBlock, rowIndex); } - updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); +// updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } @@ -1641,7 +1655,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); num = 1; memcpy(pInfo->prevData, val, bytes); @@ -1660,7 +1674,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); } tfree(pInfo->prevData); @@ -1710,8 +1724,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); pInfo->curWindow.skey = tsList[j]; pInfo->curWindow.ekey = tsList[j]; @@ -1731,8 +1745,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { @@ -1806,7 +1820,7 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx) { +static bool functionNeedToExecute(SqlFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // in case of timestamp column, always generated results. @@ -2046,6 +2060,10 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC pCtx->resDataInfo.type = pSqlExpr->resSchema.type; pCtx->order = TSDB_ORDER_ASC; + if (i == 0) { + pCtx->functionId = FUNCTION_TS; + } + // pCtx->functionId = pSqlExpr->functionId; // pCtx->stableQuery = pQueryAttr->stableQuery; pCtx->resDataInfo.intermediateBytes = pSqlExpr->interBytes; @@ -2214,7 +2232,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); // destroyUdfInfo(pRuntimeEnv->pUdfInfo); - destroyResultBuf(pRuntimeEnv->pResultBuf); + destroyDiskbasedBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); destroyTsComp(pRuntimeEnv, pQueryAttr); @@ -2340,46 +2358,23 @@ static bool isCachedLastQuery(STaskAttr *pQueryAttr) { return true; } - - -/** - * The following 4 kinds of query are treated as the tags query - * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query - */ -bool onlyQueryTags(STaskAttr* pQueryAttr) { - for(int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQueryAttr->pExpr1[i]; - - int32_t functionId = getExprFunctionId(pExprInfo); - - if (functionId != FUNCTION_TAGPRJ && - functionId != FUNCTION_TID_TAG && - (!(functionId == FUNCTION_COUNT && pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX)) && - (!(functionId == FUNCTION_PRJ && TSDB_COL_IS_UD_COL(pExprInfo->base.pColumns->flag)))) { - return false; - } - } - - return true; -} - ///////////////////////////////////////////////////////////////////////////////////////////// - -void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { - assert(key >= keyFirst && key <= keyLast && pQueryAttr->interval.sliding <= pQueryAttr->interval.interval); - win->skey = taosTimeTruncate(key, &pQueryAttr->interval, pQueryAttr->precision); +//todo refactor : return window +void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) { + assert(key >= keyFirst && key <= keyLast && pInterval->sliding <= pInterval->interval); + win->skey = taosTimeTruncate(key, pInterval, precision); /* - * if the realSkey > INT64_MAX - pQueryAttr->interval.interval, the query duration between + * if the realSkey > INT64_MAX - pInterval->interval, the query duration between * realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges. */ - if (keyFirst > (INT64_MAX - pQueryAttr->interval.interval)) { - assert(keyLast - keyFirst < pQueryAttr->interval.interval); + if (keyFirst > (INT64_MAX - pInterval->interval)) { + assert(keyLast - keyFirst < pInterval->interval); win->ekey = INT64_MAX; - } else if (pQueryAttr->interval.intervalUnit == 'n' || pQueryAttr->interval.intervalUnit == 'y') { - win->ekey = taosTimeAdd(win->skey, pQueryAttr->interval.interval, pQueryAttr->interval.intervalUnit, pQueryAttr->precision) - 1; + } else if (pInterval->intervalUnit == 'n' || pInterval->intervalUnit == 'y') { + win->ekey = taosTimeAdd(win->skey, pInterval->interval, pInterval->intervalUnit, precision) - 1; } else { - win->ekey = win->skey + pQueryAttr->interval.interval - 1; + win->ekey = win->skey + pInterval->interval - 1; } } @@ -2605,7 +2600,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); if (QUERY_IS_ASC_QUERY(pQueryAttr)) { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); assert(w.ekey >= pBlockInfo->window.skey); if (w.ekey < pBlockInfo->window.ekey) { @@ -2613,7 +2608,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } while(1) { - getNextTimeWindow(pQueryAttr, &w); +// getNextTimeWindow(pQueryAttr, &w); if (w.skey > pBlockInfo->window.ekey) { break; } @@ -2624,7 +2619,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } } } else { - getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); assert(w.skey <= pBlockInfo->window.ekey); if (w.skey > pBlockInfo->window.skey) { @@ -2632,7 +2627,7 @@ static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockI } while(1) { - getNextTimeWindow(pQueryAttr, &w); +// getNextTimeWindow(pQueryAttr, &w); if (w.ekey < pBlockInfo->window.skey) { break; } @@ -2863,8 +2858,6 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, tfree(p); } - - static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t numOfTags, int16_t colId); static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); @@ -3379,10 +3372,8 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } - -void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int32_t stage, SExecTaskInfo* pTaskInfo) { - SOptrBasicInfo *pInfo = &pAggInfo->binfo; - +// TODO refactor: some function move away +void setDefaultOutputBuf_rv(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t stage, SExecTaskInfo* pTaskInfo) { SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; @@ -3390,9 +3381,7 @@ void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int32_t stage, SExecTask int64_t tid = 0; int64_t groupId = 0; - - pAggInfo->keyBuf = realloc(pAggInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pAggInfo); + SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3540,9 +3529,6 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCt } void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t numOfOutput = pOperator->numOfOutput; // if (pQueryAttr->groupbyColumn || QUERY_IS_INTERVAL_QUERY(pQueryAttr) || pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow) { // // for each group result, call the finalize function for each column @@ -3605,7 +3591,7 @@ STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow // set more initial size of interval/groupby query // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { int32_t initialSize = 128; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -3623,7 +3609,7 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { // set more initial size of interval/groupby query int32_t initialSize = 16; - int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); + int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); if (code != TSDB_CODE_SUCCESS) { tfree(pTableQueryInfo); return NULL; @@ -3716,7 +3702,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, i int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; SResultRow* pResultRow = - doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, pAggInfo); + doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, &pAggInfo->aggSup); assert (pResultRow != NULL); /* @@ -3935,7 +3921,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { TSKEY sk = TMIN(win.skey, win.ekey); TSKEY ek = TMAX(win.skey, win.ekey); - getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { // if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { @@ -4520,18 +4506,23 @@ void queryCostStatis(SExecTaskInfo *pTaskInfo) { // return true; //} -void appendDownstream(SOperatorInfo* p, SOperatorInfo* pUpstream) { +int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { if (p->pDownstream == NULL) { assert(p->numOfDownstream == 0); } - p->pDownstream = realloc(p->pDownstream, POINTER_BYTES * (p->numOfDownstream + 1)); - p->pDownstream[p->numOfDownstream++] = pUpstream; + p->pDownstream = calloc(1, num * POINTER_BYTES); + if (p->pDownstream == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES); + p->numOfDownstream = num; + return TSDB_CODE_SUCCESS; } static void doDestroyTableQueryInfo(STableGroupInfo* pTableqinfoGroupInfo); -void createResultBlock(const SArray* pExprInfo, SExchangeInfo* pInfo, const SOperatorInfo* pOperator, size_t size); static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_t qId, bool isSTableQuery) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 @@ -4624,11 +4615,11 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr tsBufSetTraverseOrder(pRuntimeEnv->pTsBuf, order); } - int32_t ps = DEFAULT_PAGE_SIZE; + int32_t ps = 4096; getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp"); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5441,13 +5432,17 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); for(int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - taosArrayPush(pColList, &pExpr->pExpr->pSchema[0].colId); } // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); + int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); + if (code != 0) { + tfree(pInfo); + tfree(pOperator); + return NULL; + } pInfo->readerHandle = streamReadHandle; @@ -5462,7 +5457,6 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp return pOperator; } - void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInfo* pDownstream) { assert(pTableScanInfo != NULL && pDownstream != NULL); @@ -5595,14 +5589,20 @@ SArray* getResultGroupCheckColumns(STaskAttr* pQuery) { return pOrderColumns; } -static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { - SMultiwayMergeInfo *pInfo = (SMultiwayMergeInfo*) param; - destroyBasicOperatorInfo(&pInfo->binfo, numOfOutput); +static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo); +static void clearupAggSup(SAggSupporter* pAggSup); - taosArrayDestroy(pInfo->orderColumnList); - taosArrayDestroy(pInfo->groupColumnList); - tfree(pInfo->prevRow); - tfree(pInfo->currentGroupColData); +static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { + SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*) param; + taosArrayDestroy(pInfo->orderInfo); + taosArrayDestroy(pInfo->groupInfo); + + if (pInfo->pSortHandle != NULL) { + tsortDestroySortHandle(pInfo->pSortHandle); + } + blockDataDestroy(pInfo->binfo.pRes); + + clearupAggSup(&pInfo->aggSup); } static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { @@ -5612,487 +5612,396 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevRow); } -SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, - SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->resultRowFactor = - (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, pRuntimeEnv->pQueryAttr->topBotQuery, false)); +static SExprInfo* exprArrayDup(SArray* pExprList) { + size_t numOfOutput = taosArrayGetSize(pExprList); - pRuntimeEnv->scanFlag = MERGE_STAGE; // TODO init when creating pCtx - - pInfo->multiGroupResults = groupResultMixedUp; - pInfo->pMerge = param; - pInfo->bufCapacity = 4096; - pInfo->udfInfo = pUdfInfo; - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor); - pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); - pInfo->orderColumnList = getOrderCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->groupColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - - // TODO refactor - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { -// len += pExpr[i].base.; + SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); + for (int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprList, i); + assignExprInfo(&p[i], pExpr); } - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); - int32_t offset = POINTER_BYTES * numOfCols; + return p; +} - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; +// TODO merge aggregate super table +static void appendOneRowToDataBlock(SSDataBlock *pBlock, STupleHandle* pTupleHandle) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); - offset += pExpr[index->colIndex].base.resSchema.bytes; + bool isNull = tsortIsNullVal(pTupleHandle, i); + if (isNull) { + colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + } else { + char* pData = tsortGetValue(pTupleHandle, i); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } } - numOfCols = (pInfo->groupColumnList != NULL)? (int32_t)taosArrayGetSize(pInfo->groupColumnList):0; - pInfo->currentGroupColData = calloc(1, (POINTER_BYTES * numOfCols + len)); - offset = POINTER_BYTES * numOfCols; + pBlock->info.rows += 1; +} - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->currentGroupColData[i] = (char*)pInfo->currentGroupColData + offset; +static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, bool hasVarCol, int32_t capacity) { + blockDataClearup(pDataBlock, hasVarCol); - SColIndex* index = taosArrayGet(pInfo->groupColumnList, i); - offset += pExpr[index->colIndex].base.resSchema.bytes; - } + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + appendOneRowToDataBlock(pDataBlock, pTupleHandle); + if (pDataBlock->info.rows >= capacity) { + return pDataBlock; + } + } - pInfo->seed = rand(); - setDefaultOutputBuf(pRuntimeEnv, &pInfo->binfo, pInfo->seed, MERGE_STAGE); + return (pDataBlock->info.rows > 0)? pDataBlock:NULL; +} - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "GlobalAggregate"; -// pOperator->operatorType = OP_GlobalAggregate; - pOperator->blockingOptr = true; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; -// pOperator->exec = doGlobalAggregate; - pOperator->cleanupFn = destroyGlobalAggOperatorInfo; - appendDownstream(pOperator, downstream); +SSDataBlock* loadNextDataBlock(void* param) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + bool newgroup = false; - return pOperator; + return pOperator->exec(pOperator, &newgroup); } -SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExprInfo *pExpr, int32_t numOfOutput, - int32_t numOfRows, void *merger) { - SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo)); - - pInfo->pMerge = merger; - pInfo->bufCapacity = numOfRows; - pInfo->orderColumnList = getResultGroupCheckColumns(pRuntimeEnv->pQueryAttr); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, numOfRows); +static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) { + size_t size = taosArrayGetSize(groupInfo); + if (size == 0) { + return true; + } - { // todo extract method to create prev compare buffer - int32_t len = 0; - for(int32_t i = 0; i < numOfOutput; ++i) { -// len += pExpr[i].base.colBytes; - } + for (int32_t i = 0; i < size; ++i) { + int32_t* index = taosArrayGet(groupInfo, i); - int32_t numOfCols = (pInfo->orderColumnList != NULL)? (int32_t) taosArrayGetSize(pInfo->orderColumnList):0; - pInfo->prevRow = calloc(1, (POINTER_BYTES * numOfCols + len)); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); + bool isNull = colDataIsNull(pColInfo, rowIndex, pBlock->info.rows, NULL); - int32_t offset = POINTER_BYTES * numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - pInfo->prevRow[i] = (char*)pInfo->prevRow + offset; + if ((isNull && buf[i] != NULL) || (!isNull && buf[i] == NULL)) { + return false; + } - SColIndex* index = taosArrayGet(pInfo->orderColumnList, i); -// offset += pExpr[index->colIndex].base.colBytes; + char* pCell = colDataGetData(pColInfo, rowIndex); + if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + if (varDataLen(pCell) != varDataLen(buf[i])) { + return false; + } else { + if (memcmp(varDataVal(pCell), varDataVal(buf[i]), varDataLen(pCell)) != 0) { + return false; + } + } + } else { + if (memcmp(pCell, buf[i], pColInfo->info.bytes) != 0) { + return false; + } } } - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "MultiwaySortOperator"; -// pOperator->operatorType = OP_MultiwayMergeSort; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = numOfOutput; - pOperator->pExpr = pExpr; -// pOperator->exec = doMultiwayMergeSort; - pOperator->cleanupFn = destroyGlobalAggOperatorInfo; - return pOperator; + return 0; } -typedef struct SExternalMemSource { - SArray* pageIdList; - int32_t pageIndex; - int32_t sourceId; - int32_t rowIndex; - SSDataBlock *pBlock; -} SExternalMemSource; - -int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { - int32_t pLeftIdx = *(int32_t *)pLeft; - int32_t pRightIdx = *(int32_t *)pRight; - - SMsortComparParam *pParam = (SMsortComparParam *)param; - - SArray *pInfo = pParam->orderInfo; - - SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; - SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; - - // this input is exhausted, set the special value to denote this - if (pLeftSource->rowIndex == -1) { - return 1; +static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx *pCtx, int32_t numOfExpr, int32_t rowIndex) { + for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index + pCtx[j].startRow = rowIndex; } - if (pRightSource->rowIndex == -1) { - return -1; + for (int32_t j = 0; j < numOfExpr; ++j) { + int32_t functionId = pCtx[j].functionId; + pCtx[j].fpSet->addInput(&pCtx[j]); + +// if (functionId < 0) { +// SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); +// doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); +// } else { +// assert(!TSDB_FUNC_IS_SCALAR(functionId)); +// aAggs[functionId].mergeFunc(&pCtx[j]); +// } } +} - SSDataBlock* pLeftBlock = pLeftSource->pBlock; - SSDataBlock* pRightBlock = pRightSource->pBlock; +static void doFinalizeResultImpl(SqlFunctionCtx *pCtx, int32_t numOfExpr) { + for(int32_t j = 0; j < numOfExpr; ++j) { + int32_t functionId = pCtx[j].functionId; + // if (functionId == FUNC_TAG_DUMMY || functionId == FUNC_TS_DUMMY) { + // continue; + // } - for(int32_t i = 0; i < pInfo->size; ++i) { - SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + // if (functionId < 0) { + // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); + // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); + // } else { + pCtx[j].fpSet->addInput(&pCtx[j]); + } +} - SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); +static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) { + int32_t size = (int32_t) taosArrayGetSize(pColumnList); - bool leftNull = false; - if (pLeftColInfoData->hasNull) { - leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->rowIndex, pLeftBlock->pBlockAgg); - } + for(int32_t i = 0; i < size; ++i) { + int32_t* index = taosArrayGet(pColumnList, i); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, *index); - SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); - bool rightNull = false; - if (pRightColInfoData->hasNull) { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->rowIndex, pRightBlock->pBlockAgg); - } + char* data = colDataGetData(pColInfo, rowIndex); + memcpy(rowColData[i], data, colDataGetLength(pColInfo, rowIndex)); + } - if (leftNull && rightNull) { - continue; // continue to next slot - } + return true; +} - if (rightNull) { - return pParam->nullFirst? 1:-1; - } +static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) { + SSortedMergeOperatorInfo* pInfo = pOperator->info; - if (leftNull) { - return pParam->nullFirst? -1:1; - } + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + pCtx[i].size = 1; + } - void* left1 = colDataGet(pLeftColInfoData, pLeftSource->rowIndex); - void* right1 = colDataGet(pRightColInfoData, pRightSource->rowIndex); + for(int32_t i = 0; i < pBlock->info.rows; ++i) { + if (!pInfo->hasGroupVal) { + ASSERT(i == 0); + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); + } else { + if (needToMerge(pBlock, pInfo->groupInfo, pInfo->groupVal, i)) { + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + } else { + doFinalizeResultImpl(pCtx, numOfExpr); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); - switch(pLeftColInfoData->info.type) { - case TSDB_DATA_TYPE_INT: { - int32_t leftv = *(int32_t*)left1; - int32_t rightv = *(int32_t*)right1; + // TODO check for available buffer; - if (leftv == rightv) { - break; - } else { - if (pOrder->order == TSDB_ORDER_ASC) { - return leftv < rightv? -1 : 1; - } else { - return leftv < rightv? 1 : -1; + // next group info data + pInfo->binfo.pRes->info.rows += numOfRows; + for (int32_t j = 0; j < numOfExpr; ++j) { + if (pCtx[j].functionId < 0) { + continue; } + + pCtx[j].fpSet->addInput(&pCtx[j]); } + + doMergeResultImpl(pInfo, pCtx, numOfExpr, i); + pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); } - default: - assert(0); } } } -static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SOrderOperatorInfo* pInfo) { - /* - * load a new SDataBlock into memory of a given intermediate data-set source, - * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree - */ - if (pSource->rowIndex >= pSource->pBlock->info.rows) { - pSource->rowIndex = 0; - pSource->pageIndex += 1; - - if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { - pInfo->numOfCompleted += 1; - pSource->rowIndex = -1; - pSource->pageIndex = -1; - pSource->pBlock = blockDataDestroy(pSource->pBlock); - } else { - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); +static SSDataBlock* doMerge(SOperatorInfo* pOperator) { + SSortedMergeOperatorInfo* pInfo = pOperator->info; + SSortHandle* pHandle = pInfo->pSortHandle; - SFilePage* pPage = getBufPage(pInfo->pSortInternalBuf, getPageId(pPgInfo)); - int32_t code = blockDataFromBuf(pSource->pBlock, pPage->data); - if (code != TSDB_CODE_SUCCESS) { - return code; + SSDataBlock* pDataBlock = createOneDataBlock(pInfo->binfo.pRes); + blockDataEnsureCapacity(pDataBlock, pInfo->binfo.capacity); + + while(1) { + + blockDataClearup(pDataBlock, pInfo->hasVarCol); + while (1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; } - releaseBufPage(pInfo->pSortInternalBuf, pPage); + // build datablock for merge for one group + appendOneRowToDataBlock(pDataBlock, pTupleHandle); + if (pDataBlock->info.rows >= pInfo->binfo.capacity) { + break; + } } - } - /* - * Adjust loser tree otherwise, according to new candidate data - * if the loser tree is rebuild completed, we do not need to adjust - */ - int32_t leafNodeIndex = tMergeTreeGetAdjustIndex(pTree); - -#ifdef _DEBUG_VIEW - printf("before adjust:\t"); - tMergeTreePrint(pTree); -#endif + if (pDataBlock->info.rows == 0) { + break; + } - tMergeTreeAdjust(pTree, leafNodeIndex); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pDataBlock, TSDB_ORDER_ASC); + // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, pOperator->pRuntimeEnv, true); + doMergeImpl(pOperator, pOperator->numOfOutput, pDataBlock); + // flush to tuple store, and after all data have been handled, return to upstream node or sink node + } -#ifdef _DEBUG_VIEW - printf("\nafter adjust:\t"); - tMergeTreePrint(pTree); -#endif -} + doFinalizeResultImpl(pInfo->binfo.pCtx, pOperator->numOfOutput); + int32_t numOfRows = getNumOfResult(pInfo->binfo.pCtx, pOperator->numOfOutput); + // setTagValueForMultipleRows(pCtx, pOperator->numOfOutput, numOfRows); -static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + // TODO check for available buffer; - SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); - bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); + // next group info data + pInfo->binfo.pRes->info.rows += numOfRows; + return (pInfo->binfo.pRes->info.rows > 0)? pInfo->binfo.pRes:NULL; +} - if (isNull) { - colDataAppend(pColInfo, pBlock->info.rows, NULL, true); - } else { - char* pData = colDataGet(pSrcColInfo, *rowIndex); - colDataAppend(pColInfo, pBlock->info.rows, pData, false); - } +static SSDataBlock* doSortedMerge(void* param, bool* newgroup) { + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; } - pBlock->info.rows += 1; - *rowIndex += 1; -} - -static int32_t doAddNewSource(SOrderOperatorInfo* pInfo, SArray* pAllSources, int32_t numOfCols) { - SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); - if (pSource == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - pSource->pageIdList = getDataBufPagesIdList(pInfo->pSortInternalBuf, pInfo->sourceId); - pSource->sourceId = pInfo->sourceId; - - pSource->pBlock = calloc(1, sizeof(SSDataBlock)); - pSource->pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pSource->pBlock->info.numOfCols = numOfCols; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - SColumnInfoData* p = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); - colInfo.info = p->info; - taosArrayPush(pSource->pBlock->pDataBlock, &colInfo); + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSortedMergeOperatorInfo* pInfo = pOperator->info; + if (pOperator->status == OP_RES_TO_RETURN) { + return getSortedBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pInfo->hasVarCol, pInfo->binfo.capacity); } - taosArrayPush(pAllSources, &pSource); - - pInfo->sourceId += 1; + SSchema* p = blockDataExtractSchema(pInfo->binfo.pRes, NULL); + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, + numOfBufPage, p, pInfo->binfo.pRes->info.numOfCols, "GET_TASKID(pTaskInfo)"); - int32_t rowSize = blockDataGetSerialRowSize(pSource->pBlock); - int32_t numOfRows = (getBufPageSize(pInfo->pSortInternalBuf) - blockDataGetSerialMetaSize(pInfo->pDataBlock))/rowSize; - - return blockDataEnsureCapacity(pSource->pBlock, numOfRows); -} - -void addToDiskbasedBuf(SOrderOperatorInfo* pInfo, SArray* pSources, jmp_buf env) { - int32_t start = 0; - - while(start < pInfo->pDataBlock->info.rows) { - int32_t stop = 0; - blockDataSplitRows(pInfo->pDataBlock, pInfo->hasVarCol, start, &stop, getBufPageSize(pInfo->pSortInternalBuf)); - SSDataBlock* p = blockDataExtractBlock(pInfo->pDataBlock, start, stop - start + 1); - if (p == NULL) { - longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - int32_t pageId = -1; - SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); - if (pPage == NULL) { - assert(0); - longjmp(env, terrno); - } - - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); - assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); - - blockDataToBuf(pPage->data, p); - - setBufPageDirty(pPage, true); - releaseBufPage(pInfo->pSortInternalBuf, pPage); + tfree(p); + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - blockDataDestroy(p); - start = stop + 1; + for(int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + ps->param = pOperator->pDownstream[i]; + tsortAddSource(pInfo->pSortHandle, ps); } - int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; - blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); - - int32_t code = doAddNewSource(pInfo, pSources, numOfCols); + int32_t code = tsortOpen(pInfo->pSortHandle); if (code != TSDB_CODE_SUCCESS) { - longjmp(env, code); + longjmp(pTaskInfo->env, terrno); } + + pOperator->status = OP_RES_TO_RETURN; + return doMerge(pOperator); } -static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SDiskbasedBuf* pBuf) { - cmpParam->pSources = taosArrayGet(pSources, startIndex); - cmpParam->numOfSources = (endIndex - startIndex + 1); +static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { + SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; - SPageInfo* pPgInfo = *(SPageInfo**) taosArrayGet(pSource->pageIdList, pSource->pageIndex); + size_t numOfOrder = taosArrayGetSize(pOrderVal); + for (int32_t j = 0; j < numOfOrder; ++j) { + SBlockOrderInfo orderInfo = {0}; + SOrder* pOrder = taosArrayGet(pOrderVal, j); + orderInfo.order = pOrder->order; - SFilePage* pPage = getBufPage(pBuf, getPageId(pPgInfo)); - int32_t code = blockDataFromBuf(cmpParam->pSources[i]->pBlock, pPage->data); - if (code != TSDB_CODE_SUCCESS) { - return code; + for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + if (pExpr->base.resSchema.colId == pOrder->col.info.colId) { + orderInfo.colIndex = i; + break; + } } - releaseBufPage(pBuf, pPage); + taosArrayPush(pOrderInfo, &orderInfo); } - return TSDB_CODE_SUCCESS; + return pOrderInfo; } -static int32_t sortComparClearup(SMsortComparParam* cmpParam) { - for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { - SExternalMemSource* pSource = cmpParam->pSources[i]; - blockDataDestroy(pSource->pBlock); - tfree(pSource); +static int32_t initGroupCol(SArray* pExprInfo, SArray* pGroupInfo, SSortedMergeOperatorInfo* pInfo) { + if (pGroupInfo == NULL || taosArrayGetSize(pGroupInfo) == 0) { + return 0; } - cmpParam->numOfSources = 0; -} - -static SSDataBlock* getSortedBlockData(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo, SMsortComparParam* cmpParam, int32_t capacity) { - blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); - - while(1) { - if (cmpParam->numOfSources == pInfo->numOfCompleted) { - break; - } - - int32_t index = tMergeTreeGetChosenIndex(pInfo->pMergeTree); - - SExternalMemSource *pSource = (*cmpParam).pSources[index]; - appendOneRowToDataBlock(pInfo->pDataBlock, pSource->pBlock, &pSource->rowIndex); - - int32_t code = adjustMergeTreeForNextTuple(pSource, pInfo->pMergeTree, pInfo); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } - - if (pInfo->pDataBlock->info.rows >= capacity) { - return pInfo->pDataBlock; + int32_t len = 0; + SArray* plist = taosArrayInit(3, sizeof(SColumn)); + pInfo->groupInfo = taosArrayInit(3, sizeof(int32_t)); + + if (plist == NULL || pInfo->groupInfo == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + size_t numOfGroupCol = taosArrayGetSize(pInfo->groupInfo); + for(int32_t i = 0; i < numOfGroupCol; ++i) { + SColumn* pCol = taosArrayGet(pGroupInfo, i); + for(int32_t j = 0; j < taosArrayGetSize(pExprInfo); ++j) { + SExprInfo* pe = taosArrayGet(pExprInfo, j); + if (pe->base.resSchema.colId == pCol->info.colId) { + taosArrayPush(plist, pCol); + taosArrayPush(pInfo->groupInfo, &j); + len += pCol->info.bytes; + break; + } } } - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; -} - -static int32_t doInternalSort(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo) { - size_t numOfSources = taosArrayGetSize(pInfo->pSources); - - // Calculate the I/O counts to complete the data sort. - double sortCount = floorl(log2(numOfSources) / log2(getNumOfInMemBufPages(pInfo->pSortInternalBuf))); - - pInfo->totalElapsed = taosGetTimestampUs() - pInfo->startTs; - qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, - GET_TASKID(pTaskInfo), (int32_t) (sortCount + 1), getTotalBufSize(pInfo->pSortInternalBuf), pInfo->sortElapsed, - pInfo->totalElapsed); - - size_t pgSize = getBufPageSize(pInfo->pSortInternalBuf); - int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pInfo->pDataBlock))/ blockDataGetSerialRowSize(pInfo->pDataBlock); - - blockDataEnsureCapacity(pInfo->pDataBlock, numOfRows); - - size_t numOfSorted = taosArrayGetSize(pInfo->pSources); - for(int32_t t = 0; t < sortCount; ++t) { - int64_t st = taosGetTimestampUs(); - - SArray* pResList = taosArrayInit(4, POINTER_BYTES); - SMsortComparParam resultParam = {.orderInfo = pInfo->cmpParam.orderInfo}; + ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - int32_t numOfInputSources = getNumOfInMemBufPages(pInfo->pSortInternalBuf); - int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources; - - // Only *numOfInputSources* can be loaded into buffer to perform the external sort. - for(int32_t i = 0; i < sortGroup; ++i) { - pInfo->sourceId += 1; - - int32_t end = (i + 1) * numOfInputSources - 1; - if (end > numOfSorted - 1) { - end = numOfSorted - 1; - } - - pInfo->cmpParam.numOfSources = end - i * numOfInputSources + 1; - - int32_t code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, i * numOfInputSources, end, pInfo->pSortInternalBuf); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + pInfo->groupVal = calloc(1, (POINTER_BYTES * numOfGroupCol + len)); + if (pInfo->groupVal == NULL) { + taosArrayDestroy(plist); + return TSDB_CODE_OUT_OF_MEMORY; + } - code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + int32_t offset = 0; + char *start = (char*)(pInfo->groupVal + (POINTER_BYTES * numOfGroupCol)); + for(int32_t i = 0; i < numOfGroupCol; ++i) { + pInfo->groupVal[i] = start + offset; + SColumn* pCol = taosArrayGet(plist, i); + offset += pCol->info.bytes; + } - while (1) { - SSDataBlock* pDataBlock = getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, numOfRows); - if (pDataBlock == NULL) { - break; - } + taosArrayDestroy(plist); - int32_t pageId = -1; - SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); - if (pPage == NULL) { - assert(0); - longjmp(pTaskInfo->env, terrno); - } + return TSDB_CODE_SUCCESS; +} - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); - assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo) { + SSortedMergeOperatorInfo* pInfo = calloc(1, sizeof(SSortedMergeOperatorInfo)); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } - blockDataToBuf(pPage->data, pDataBlock); + int32_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - setBufPageDirty(pPage, true); - releaseBufPage(pInfo->pSortInternalBuf, pPage); + if (pInfo->binfo.pCtx == NULL || pInfo->binfo.pRes == NULL) { + goto _error; + } - blockDataClearup(pDataBlock, pInfo->hasVarCol); - } + int32_t code = initAggSup(&pInfo->aggSup, pExprInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - tMergeTreeDestroy(pInfo->pMergeTree); - pInfo->numOfCompleted = 0; + setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); + code = initGroupCol(pExprInfo, pGroupInfo, pInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - code = doAddNewSource(pInfo, pResList, pInfo->pDataBlock->info.numOfCols); - if (code != 0) { - longjmp(pTaskInfo->env, code); - } - } +// pInfo->resultRowFactor = (int32_t)(getRowNumForMultioutput(pRuntimeEnv->pQueryAttr, +// pRuntimeEnv->pQueryAttr->topBotQuery, false)); + pInfo->sortBufSize = 1024 * 16; // 1MB + pInfo->bufPageSize = 1024; + pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); - sortComparClearup(&pInfo->cmpParam); + pInfo->binfo.capacity = blockDataGetCapacityInRow(pInfo->binfo.pRes, pInfo->bufPageSize); - taosArrayClear(pInfo->pSources); - taosArrayAddAll(pInfo->pSources, pResList); - taosArrayDestroy(pResList); + pOperator->name = "SortedMerge"; + pOperator->operatorType = OP_SortedMerge; + pOperator->blockingOptr = true; + pOperator->status = OP_IN_EXECUTING; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = exprArrayDup(pExprInfo); - pInfo->cmpParam = resultParam; - numOfSorted = taosArrayGetSize(pInfo->pSources); + pOperator->pTaskInfo = pTaskInfo; + pOperator->exec = doSortedMerge; + pOperator->cleanupFn = destroySortedMergeOperatorInfo; - int64_t el = taosGetTimestampUs() - st; - pInfo->totalElapsed += el; + code = appendDownstream(pOperator, downstream, numOfDownstream); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - SDiskbasedBufStatis statis = getDBufStatis(pInfo->pSortInternalBuf); + return pOperator; - qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", GET_TASKID(pTaskInfo), t + 1, el, statis.loadBytes/1024.0, - statis.flushBytes/1024.0); + _error: + if (pInfo != NULL) { + destroySortedMergeOperatorInfo(pInfo, numOfOutput); } - pInfo->cmpParam.numOfSources = taosArrayGetSize(pInfo->pSources); - return 0; + tfree(pInfo); + tfree(pOperator); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; } static SSDataBlock* doSort(void* param, bool* newgroup) { @@ -6103,103 +6012,33 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SOrderOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pBlock = NULL; - if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); + return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes); } - int64_t st = taosGetTimestampUs(); - - while(1) { - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); - pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); - publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC); - - // start to flush data into disk and try do multiway merge sort - if (pBlock == NULL) { - break; - } + SSchema* p = blockDataExtractSchema(pInfo->pDataBlock, NULL); + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->orderInfo, pInfo->nullFirst, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, + numOfBufPage, p, pInfo->pDataBlock->info.numOfCols, "GET_TASKID(pTaskInfo)"); - int32_t code = blockDataMerge(pInfo->pDataBlock, pBlock); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pTaskInfo->env, code); - } - - size_t size = blockDataGetSize(pInfo->pDataBlock); - if (size > pInfo->sortBufSize) { - // Perform the in-memory sort and then flush data in the buffer into disk. - int64_t p = taosGetTimestampUs(); - blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); - - int64_t el = taosGetTimestampUs() - p; - pInfo->sortElapsed += el; - - addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); - } - } - - if (pInfo->pDataBlock->info.rows > 0) { - // Perform the in-memory sort and then flush data in the buffer into disk. - blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); - - // All sorted data are resident in memory, external memory sort is not needed. - // Return to the upstream operator directly - if (isAllDataInMemBuf(pInfo->pSortInternalBuf)) { - pOperator->status = OP_EXEC_DONE; - return (pInfo->pDataBlock->info.rows == 0)? NULL:pInfo->pDataBlock; - } - - addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); - } - - doInternalSort(pTaskInfo, pInfo); - - int32_t code = blockDataEnsureCapacity(pInfo->pDataBlock, pInfo->numOfRowsInRes); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + tfree(p); + tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); - int32_t numOfSources = taosArrayGetSize(pInfo->pSources); - ASSERT(numOfSources <= getNumOfInMemBufPages(pInfo->pSortInternalBuf)); - code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, 0, numOfSources - 1, pInfo->pSortInternalBuf); - if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); - } + SGenericSource* ps = calloc(1, sizeof(SGenericSource)); + ps->param = pOperator; + tsortAddSource(pInfo->pSortHandle, ps); - code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); + // TODO set error code; + int32_t code = tsortOpen(pInfo->pSortHandle); if (code != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, code); + longjmp(pTaskInfo->env, terrno); } pOperator->status = OP_RES_TO_RETURN; - return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); + return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->hasVarCol, pInfo->numOfRowsInRes); } -static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { - SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - - size_t numOfOrder = taosArrayGetSize(pOrderVal); - for (int32_t j = 0; j < numOfOrder; ++j) { - SBlockOrderInfo orderInfo = {0}; - SOrder* pOrder = taosArrayGet(pOrderVal, j); - orderInfo.order = pOrder->order; - - for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGet(pExprInfo, i); - if (pExpr->base.resSchema.colId == pOrder->col.info.colId) { - orderInfo.colIndex = i; - break; - } - } - - taosArrayPush(pOrderInfo, &orderInfo); - } - - return pOrderInfo; -} - -SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal) { +SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo) { SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6214,8 +6053,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pInfo->numOfRowsInRes = 1024; pInfo->pDataBlock = createOutputBuf_rv(pExprInfo, pInfo->numOfRowsInRes); - pInfo->pSources = taosArrayInit(4, POINTER_BYTES); - pInfo->cmpParam.orderInfo = createBlockOrder(pExprInfo, pOrderVal); + pInfo->orderInfo = createBlockOrder(pExprInfo, pOrderVal); for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); @@ -6225,8 +6063,7 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI } } - int32_t code = createDiskbasedBuffer(&pInfo->pSortInternalBuf, pInfo->bufPageSize, pInfo->sortBufSize, 1, "/tmp/"); - if (pInfo->pSources == NULL || code != 0 || pInfo->cmpParam.orderInfo == NULL || pInfo->pDataBlock == NULL) { + if (pInfo->orderInfo == NULL || pInfo->pDataBlock == NULL) { tfree(pOperator); destroyOrderOperatorInfo(pInfo, taosArrayGetSize(pExprInfo)); tfree(pInfo); @@ -6240,10 +6077,12 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; + + pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doSort; pOperator->cleanupFn = destroyOrderOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -6537,22 +6376,19 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { return NULL; } - STableIntervalOperatorInfo* pIntervalInfo = pOperator->info; + STableIntervalOperatorInfo* pInfo = pOperator->info; - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(pAggInfo->pGroupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes; + return pInfo->binfo.pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - STimeWindow win = pQueryAttr->window; - +// int32_t order = pQueryAttr->order.order; +// STimeWindow win = pQueryAttr->window; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { @@ -6564,30 +6400,30 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { break; } -// setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, TSDB_ORDER_ASC); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value - pQueryAttr->order.order = order; - pQueryAttr->window = win; +// pQueryAttr->order.order = order; +// pQueryAttr->window = win; pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); + closeAllResultRows(&pInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + toSDatablock(&pInfo->groupResInfo, pInfo->pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; + return pInfo->binfo.pRes->info.rows == 0? NULL:pInfo->binfo.pRes; } static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { @@ -6602,11 +6438,11 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6627,8 +6463,8 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); - hashAllIntervalAgg(pOperator, &pIntervalInfo->resultRowInfo, pBlock, 0); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); + hashAllIntervalAgg(pOperator, &pIntervalInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value @@ -6636,18 +6472,18 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pQueryAttr->window = win; pOperator->status = OP_RES_TO_RETURN; - closeAllResultRows(&pIntervalInfo->resultRowInfo); + closeAllResultRows(&pIntervalInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); + finalizeQueryResult(pOperator, pIntervalInfo->binfo.pCtx, &pIntervalInfo->binfo.resultRowInfo, pIntervalInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); + initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->binfo.resultRowInfo); // toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes->info.rows == 0? NULL:pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes->info.rows == 0? NULL:pIntervalInfo->binfo.pRes; } static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { @@ -6663,14 +6499,14 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { int64_t st = taosGetTimestampUs(); // copyToSDataBlock(NULL, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } SQInfo* pQInfo = pRuntimeEnv->qinfo; pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6691,7 +6527,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); @@ -6703,11 +6539,11 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { @@ -6721,11 +6557,11 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -6746,7 +6582,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; // setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); - setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); + setInputDataBlock(pOperator, pIntervalInfo->binfo.pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); hashAllIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pTableQueryInfo->groupIndex); @@ -6759,14 +6595,14 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { int64_t st = taosGetTimestampUs(); // copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); - if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { + if (pIntervalInfo->binfo.pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } SQInfo* pQInfo = pRuntimeEnv->qinfo; pQInfo->summary.firstStageMergeTime += (taosGetTimestampUs() - st); - return pIntervalInfo->pRes; + return pIntervalInfo->binfo.pRes; } static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { @@ -6818,8 +6654,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); pInfo->curWindow.skey = tsList[j]; pInfo->curWindow.ekey = tsList[j]; @@ -6840,8 +6676,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, - pSDataBlock->info.rows, pOperator->numOfOutput); +// doApplyFunctions(pRuntimeEnv, pBInfo->pCtx, &pInfo->curWindow, pInfo->start, pInfo->numOfRows, tsList, +// pSDataBlock->info.rows, pOperator->numOfOutput); } static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { @@ -7160,18 +6996,37 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } +static int32_t initAggSup(SAggSupporter* pAggSup, SArray* pExprInfo) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + + pAggSup->keyBuf = calloc(1, sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES); + pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); + pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + pAggSup->pool = initResultRowPool(getResultRowSize(pExprInfo)); + pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); + + if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || + pAggSup->pResultRowHashTable == NULL || pAggSup->pool == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + return TSDB_CODE_SUCCESS; +} + +static void clearupAggSup(SAggSupporter* pAggSup) { + tfree(pAggSup->keyBuf); + taosHashCleanup(pAggSup->pResultRowHashTable); + taosHashCleanup(pAggSup->pResultRowListSet); + taosArrayDestroy(pAggSup->pResultRowArrayList); + destroyResultRowPool(pAggSup->pool); +} + static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t numOfRows, const STableGroupInfo* pTableGroupInfo) { pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); pInfo->binfo.capacity = 4096; - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - - pInfo->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pInfo->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); - pInfo->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); - + initAggSup(&pInfo->aggSup, pExprInfo); pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); int32_t index = 0; @@ -7193,17 +7048,6 @@ static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t n return TSDB_CODE_SUCCESS; } -static SExprInfo* exprArrayDup(SArray* pExprInfo) { - size_t numOfOutput = taosArrayGetSize(pExprInfo); - SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); - for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); - assignExprInfo(&p[i], pExpr); - } - - return p; -} - SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); @@ -7211,7 +7055,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE //(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); - setDefaultOutputBuf_rv(pInfo, MAIN_SCAN, pTaskInfo); + setDefaultOutputBuf_rv(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; @@ -7225,7 +7069,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; pOperator->cleanupFn = destroyAggOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7285,10 +7129,7 @@ static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); - taosArrayDestroy(pInfo->cmpParam.orderInfo); - destroyResultBuf(pInfo->pSortInternalBuf); - - tMergeTreeDestroy(pInfo->pMergeTree); + taosArrayDestroy(pInfo->orderInfo); } static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { @@ -7312,7 +7153,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); - initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; @@ -7325,7 +7166,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray pOperator->exec = doMultiTableAggregate; pOperator->cleanupFn = destroyAggOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7340,7 +7181,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity); pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset); - initResultRowInfo(&pBInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pBInfo->resultRowInfo, 8); setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -7355,7 +7196,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doProjectOperation; pOperator->cleanupFn = destroyProjectOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7413,7 +7254,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanupFn = destroyConditionOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7431,32 +7272,49 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->exec = doLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initAggSup(&pInfo->aggSup, pExprInfo); + + // todo: + pInfo->order = TSDB_ORDER_ASC; + pInfo->precision = TSDB_TIME_PRECISION_MICRO; + pInfo->win.skey = INT64_MIN; + pInfo->win.ekey = INT64_MAX; + pInfo->interval.intervalUnit = 's'; + pInfo->interval.slidingUnit = 's'; + pInfo->interval.interval = 1000; + pInfo->interval.sliding = 1000; + + int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/"); + + int32_t numOfOutput = taosArrayGetSize(pExprInfo); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); + pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity); + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TimeIntervalAggOperator"; -// pOperator->operatorType = OP_TimeWindow; + pOperator->operatorType = OP_TimeWindow; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; - pOperator->pExpr = pExpr; + pOperator->pExpr = exprArrayDup(pExprInfo); + + pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfOutput; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7464,9 +7322,9 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -7481,7 +7339,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->exec = doAllIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7491,7 +7349,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pInfo->reptScan = false; pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "StateWindowOperator"; @@ -7505,7 +7363,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->exec = doStateWindowAgg; pOperator->cleanupFn = destroyStateWindowOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { @@ -7513,7 +7371,7 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); pInfo->prevTs = INT64_MIN; pInfo->reptScan = false; @@ -7530,16 +7388,16 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = doSessionWindowAgg; pOperator->cleanupFn = destroySWindowOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableTimeIntervalOperator"; @@ -7554,16 +7412,16 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->exec = doSTableIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo)); - pInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->rowCellInfoOffset); - pInfo->pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "AllMultiTableTimeIntervalOperator"; @@ -7578,12 +7436,11 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->exec = doAllSTableIntervalAgg; pOperator->cleanupFn = destroyBasicOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } - SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); pInfo->colIndex = -1; // group by column index @@ -7597,7 +7454,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pRuntimeEnv->resultInfo.capacity); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "GroupbyAggOperator"; @@ -7611,7 +7468,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->exec = hashGroupbyAggregate; pOperator->cleanupFn = destroyGroupbyOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7627,7 +7484,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); +// getAlignQueryTimeWindow(pQueryAttr, pQueryAttr->window.skey, sk, ek, &w); pInfo->pFillInfo = taosCreateFillInfo(pQueryAttr->order.order, w.skey, 0, (int32_t)pRuntimeEnv->resultInfo.capacity, numOfOutput, @@ -7650,7 +7507,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->exec = doFill; pOperator->cleanupFn = destroySFillOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7698,7 +7555,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanupFn = destroySlimitOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -7996,7 +7853,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pExpr = pExpr; pOperator->cleanupFn = destroyDistinctOperatorInfo; - appendDownstream(pOperator, downstream); + int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; } @@ -8186,16 +8043,22 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask STableGroupInfo groupInfo = {0}; int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + SArray* idList = NULL; - SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); - ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + if (groupInfo.numOfTables > 0) { + SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); + ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); - // Transfer the Array of STableKeyInfo into uid list. - size_t numOfTables = taosArrayGetSize(pa); - SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); - for(int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); - taosArrayPush(idList, &pkeyInfo->uid); + // Transfer the Array of STableKeyInfo into uid list. + size_t numOfTables = taosArrayGetSize(pa); + idList = taosArrayInit(numOfTables, sizeof(uint64_t)); + + for (int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); + taosArrayPush(idList, &pkeyInfo->uid); + } + } else { + idList = taosArrayInit(4, sizeof(uint64_t)); } SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pPhyNode->pTargets, idList, pTaskInfo); @@ -8370,7 +8233,6 @@ int32_t buildArithmeticExprFromMsg(SExprInfo *pExprInfo, void *pQueryMsg) { return TSDB_CODE_SUCCESS; } - static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SColumnInfo* pTagCols, SExprInfo* pExprs, int32_t numOfOutput, int32_t tagLen, bool superTable) { for (int32_t i = 0; i < numOfOutput; ++i) { int16_t functId = getExprFunctionId(&pExprs[i]); @@ -8534,66 +8396,6 @@ int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters) { // return ret; } - -// todo refactor -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo *pUdfInfo) { -// *pExprInfo = NULL; -// int32_t code = TSDB_CODE_SUCCESS; -// -// SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo)); -// if (pExprs == NULL) { -// return TSDB_CODE_QRY_OUT_OF_MEMORY; -// } -// -// bool isSuperTable = QUERY_IS_STABLE_QUERY(pQueryMsg->queryType); -// -// for (int32_t i = 0; i < numOfOutput; ++i) { -// pExprs[i].base = *pExpr[i]; -// memset(pExprs[i].base.param, 0, sizeof(SVariant) * tListLen(pExprs[i].base.param)); -// -// for (int32_t j = 0; j < pExpr[i]->numOfParams; ++j) { -// taosVariantAssign(&pExprs[i].base.param[j], &pExpr[i]->param[j]); -// } -// -// pExprs[i].base.resSchema.type = 0; -// -// int16_t type = 0; -// int16_t bytes = 0; -// -// // parse the arithmetic expression -// if (pExprs[i].base.functionId == FUNCTION_ARITHM) { -// code = buildArithmeticExprFromMsg(&pExprs[i], pQueryMsg); -// -// if (code != TSDB_CODE_SUCCESS) { -// tfree(pExprs); -// return code; -// } -// -// type = TSDB_DATA_TYPE_DOUBLE; -// bytes = tDataTypes[type].bytes; -// } else { -// int32_t index = pExprs[i].base.colInfo.colIndex; -// assert(prevExpr[index].base.resSchema.colId == pExprs[i].base.pColumns->info.colId); -// -// type = prevExpr[index].base.resSchema.type; -// bytes = prevExpr[index].base.resSchema.bytes; -// } -// -// int32_t param = (int32_t)pExprs[i].base.param[0].i; -// if (getResultDataInfo(type, bytes, functionId, param, &pExprs[i].base.resSchema.type, &pExprs[i].base.resSchema.bytes, -// &pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) { -// tfree(pExprs); -// return TSDB_CODE_QRY_INVALID_MSG; -// } -// -// assert(isValidDataType(pExprs[i].base.resSchema.type)); -// } -// -// *pExprInfo = pExprs; - return TSDB_CODE_SUCCESS; -} - SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code) { if (pQueryMsg->numOfGroupCols == 0) { return NULL; @@ -9002,30 +8804,3 @@ void releaseQueryBuf(size_t numOfTables) { // restore value is not enough buffer available atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } - -void freeQueryAttr(STaskAttr* pQueryAttr) { - if (pQueryAttr != NULL) { - if (pQueryAttr->fillVal != NULL) { - tfree(pQueryAttr->fillVal); - } - - pQueryAttr->pFilterInfo = doDestroyFilterInfo(pQueryAttr->pFilterInfo, pQueryAttr->numOfFilterCols); - - pQueryAttr->pExpr1 = destroyQueryFuncExpr(pQueryAttr->pExpr1, pQueryAttr->numOfOutput); - pQueryAttr->pExpr2 = destroyQueryFuncExpr(pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); - pQueryAttr->pExpr3 = destroyQueryFuncExpr(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3); - - tfree(pQueryAttr->tagColList); - tfree(pQueryAttr->pFilterInfo); - - pQueryAttr->tableCols = freeColumnInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols); - - if (pQueryAttr->pGroupbyExpr != NULL) { - taosArrayDestroy(pQueryAttr->pGroupbyExpr->columnInfo); - tfree(pQueryAttr->pGroupbyExpr); - } - -// filterFreeInfo(pQueryAttr->pFilters); - } -} - diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c new file mode 100644 index 0000000000000000000000000000000000000000..7f3da9bcac025c52337dec4f4348d17faf7b0dea --- /dev/null +++ b/source/libs/executor/src/tlinearhash.c @@ -0,0 +1,427 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tlinearhash.h" +#include "tdef.h" +#include "taoserror.h" +#include "tpagedbuf.h" + +#define LHASH_CAP_RATIO 0.85 + +// Always located in memory +typedef struct SLHashBucket { + SArray *pPageIdList; + int32_t size; // the number of element in this entry +} SLHashBucket; + +typedef struct SLHashObj { + SDiskbasedBuf *pBuf; + _hash_fn_t hashFn; + int32_t tuplesPerPage; + SLHashBucket **pBucket; // entry list + int32_t numOfAlloc; // number of allocated bucket ptr slot + int32_t bits; // the number of bits used in hash + int32_t numOfBuckets; // the number of buckets + int64_t size; // the number of total items +} SLHashObj; + +/** + * the data struct for each hash node + * +-----------+-------+--------+ + * | SLHashNode| key | data | + * +-----------+-------+--------+ + */ +typedef struct SLHashNode { + uint16_t keyLen; + uint16_t dataLen; +} SLHashNode; + +#define GET_LHASH_NODE_KEY(_n) (((char*)(_n)) + sizeof(SLHashNode)) +#define GET_LHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen) +#define GET_LHASH_NODE_LEN(_n) (sizeof(SLHashNode) + ((SLHashNode*)(_n))->keyLen + ((SLHashNode*)(_n))->dataLen) + +static int32_t doAddNewBucket(SLHashObj* pHashObj); + +static int32_t doGetBucketIdFromHashVal(int32_t hashv, int32_t bits) { + return hashv & ((1ul << (bits)) - 1); +} + +static int32_t doGetAlternativeBucketId(int32_t bucketId, int32_t bits, int32_t numOfBuckets) { + int32_t v = bucketId - (1ul << (bits - 1)); + ASSERT(v < numOfBuckets); + return v; +} + +static int32_t doGetRelatedSplitBucketId(int32_t bucketId, int32_t bits) { + int32_t splitBucketId = (1ul << (bits - 1)) ^ bucketId; + return splitBucketId; +} + +static void doCopyObject(char* p, const void* key, int32_t keyLen, const void* data, int32_t size) { + *(uint16_t*) p = keyLen; + p += sizeof(uint16_t); + *(uint16_t*) p = size; + p += sizeof(uint16_t); + + memcpy(p, key, keyLen); + p += keyLen; + + memcpy(p, data, size); +} + +static int32_t doAddToBucket(SLHashObj* pHashObj, SLHashBucket* pBucket, int32_t index, const void* key, int32_t keyLen, + const void* data, int32_t size) { + int32_t pageId = *(int32_t*)taosArrayGetLast(pBucket->pPageIdList); + + SFilePage* pPage = getBufPage(pHashObj->pBuf, pageId); + ASSERT (pPage != NULL); + + // put to current buf page + size_t nodeSize = sizeof(SLHashNode) + keyLen + size; + ASSERT(nodeSize + sizeof(SFilePage) <= getBufPageSize(pHashObj->pBuf)); + + if (pPage->num + nodeSize > getBufPageSize(pHashObj->pBuf)) { + releaseBufPage(pHashObj->pBuf, pPage); + + // allocate the overflow buffer page to hold this k/v. + int32_t newPageId = -1; + SFilePage* pNewPage = getNewBufPage(pHashObj->pBuf, 0, &newPageId); + if (pNewPage == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + taosArrayPush(pBucket->pPageIdList, &newPageId); + + doCopyObject(pNewPage->data, key, keyLen, data, size); + pNewPage->num = sizeof(SFilePage) + nodeSize; + + setBufPageDirty(pNewPage, true); + releaseBufPage(pHashObj->pBuf, pNewPage); + } else { + char* p = (char*) pPage + pPage->num; + doCopyObject(p, key, keyLen, data, size); + pPage->num += nodeSize; + setBufPageDirty(pPage, true); + releaseBufPage(pHashObj->pBuf, pPage); + } + + pBucket->size += 1; +// printf("===> add to bucket:0x%x, num:%d, key:%d\n", index, pBucket->size, *(int*) key); + + return TSDB_CODE_SUCCESS; +} + +static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket* pBucket) { + ASSERT(pPage != NULL && pNode != NULL && pBucket->size >= 1); + + int32_t len = GET_LHASH_NODE_LEN(pNode); + char* p = (char*) pNode + len; + + char* pEnd = (char*)pPage + pPage->num; + memmove(pNode, p, (pEnd - p)); + + pPage->num -= len; + if (pPage->num == 0) { + // this page is empty, could be recycle in the future. + } + + setBufPageDirty(pPage, true); + + pBucket->size -= 1; +} + +static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) { + size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList); + if (numOfPages <= 1) { + return; + } + + int32_t* firstPage = taosArrayGet(pBucket->pPageIdList, 0); + SFilePage* pFirst = getBufPage(pHashObj->pBuf, *firstPage); + + int32_t* pageId = taosArrayGetLast(pBucket->pPageIdList); + SFilePage* pLast = getBufPage(pHashObj->pBuf, *pageId); + + if (pLast->num <= sizeof(SFilePage)) { + // this is empty + dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); + return; + } + + char* pStart = pLast->data; + int32_t nodeSize = GET_LHASH_NODE_LEN(pStart); + while (1) { + if (pFirst->num + nodeSize < getBufPageSize(pHashObj->pBuf)) { + char* p = ((char*)pFirst) + pFirst->num; + + SLHashNode* pNode = (SLHashNode*)pStart; + doCopyObject(p, GET_LHASH_NODE_KEY(pStart), pNode->keyLen, GET_LHASH_NODE_DATA(pStart), pNode->dataLen); + + setBufPageDirty(pFirst, true); + setBufPageDirty(pLast, true); + + ASSERT(pLast->num >= nodeSize + sizeof(SFilePage)); + + pFirst->num += nodeSize; + pLast->num -= nodeSize; + + pStart += nodeSize; + if (pLast->num <= sizeof(SFilePage)) { + // this is empty + dBufSetBufPageRecycled(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + taosArrayRemove(pBucket->pPageIdList, numOfPages - 1); + break; + } + + nodeSize = GET_LHASH_NODE_LEN(pStart); + } else { // move to the front of pLast page + if (pStart != pLast->data) { + memmove(pLast->data, pStart, (((char*)pLast) + pLast->num - pStart)); + setBufPageDirty(pLast, true); + } + + releaseBufPage(pHashObj->pBuf, pLast); + releaseBufPage(pHashObj->pBuf, pFirst); + break; + } + } +} + +static int32_t doAddNewBucket(SLHashObj* pHashObj) { + if (pHashObj->numOfBuckets + 1 > pHashObj->numOfAlloc) { + int32_t newLen = pHashObj->numOfAlloc * 1.25; + if (newLen == pHashObj->numOfAlloc) { + newLen += 4; + } + + char* p = realloc(pHashObj->pBucket, POINTER_BYTES * newLen); + if (p == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memset(p + POINTER_BYTES * pHashObj->numOfBuckets, 0, newLen - pHashObj->numOfBuckets); + pHashObj->pBucket = (SLHashBucket**) p; + pHashObj->numOfAlloc = newLen; + } + + SLHashBucket* pBucket = calloc(1, sizeof(SLHashBucket)); + pHashObj->pBucket[pHashObj->numOfBuckets] = pBucket; + + pBucket->pPageIdList = taosArrayInit(2, sizeof(int32_t)); + if (pBucket->pPageIdList == NULL || pBucket == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t pageId = -1; + SFilePage* p = getNewBufPage(pHashObj->pBuf, 0, &pageId); + p->num = sizeof(SFilePage); + setBufPageDirty(p, true); + + releaseBufPage(pHashObj->pBuf, p); + taosArrayPush(pBucket->pPageIdList, &pageId); + + pHashObj->numOfBuckets += 1; +// printf("---------------add new bucket, id:0x%x, total:%d\n", pHashObj->numOfBuckets - 1, pHashObj->numOfBuckets); + return TSDB_CODE_SUCCESS; +} + +SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_t numOfTuplePerPage) { + SLHashObj* pHashObj = calloc(1, sizeof(SLHashObj)); + if (pHashObj == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int32_t code = createDiskbasedBuf(&pHashObj->pBuf, pageSize, inMemPages * pageSize, 0, "/tmp"); + if (code != 0) { + terrno = code; + return NULL; + } + + setBufPageCompressOnDisk(pHashObj->pBuf, false); + + /** + * The number of bits in the hash value, which is used to decide the exact bucket where the object should be located in. + * The initial value is 0. + */ + pHashObj->bits = 0; + pHashObj->hashFn = fn; + pHashObj->tuplesPerPage = numOfTuplePerPage; + + pHashObj->numOfAlloc = 4; // initial allocated array list + pHashObj->pBucket = calloc(pHashObj->numOfAlloc, POINTER_BYTES); + + code = doAddNewBucket(pHashObj); + if (code != TSDB_CODE_SUCCESS) { + destroyDiskbasedBuf(pHashObj->pBuf); + tfree(pHashObj); + terrno = code; + return NULL; + } + + return pHashObj; +} + +void* tHashCleanup(SLHashObj* pHashObj) { + destroyDiskbasedBuf(pHashObj->pBuf); + for(int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { + taosArrayDestroy(pHashObj->pBucket[i]->pPageIdList); + tfree(pHashObj->pBucket[i]); + } + + tfree(pHashObj->pBucket); + tfree(pHashObj); + return NULL; +} + +int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data, size_t size) { + ASSERT(pHashObj != NULL && key != NULL); + + if (pHashObj->bits == 0) { + SLHashBucket* pBucket = pHashObj->pBucket[0]; + doAddToBucket(pHashObj, pBucket, 0, key, keyLen, data, size); + } else { + int32_t hashVal = pHashObj->hashFn(key, keyLen); + int32_t v = doGetBucketIdFromHashVal(hashVal, pHashObj->bits); + + if (v >= pHashObj->numOfBuckets) { + int32_t newBucketId = doGetAlternativeBucketId(v, pHashObj->bits, pHashObj->numOfBuckets); +// printf("bucketId: 0x%x not exists, put it into 0x%x instead\n", v, newBucketId); + v = newBucketId; + } + + SLHashBucket* pBucket = pHashObj->pBucket[v]; + int32_t code = doAddToBucket(pHashObj, pBucket, v, key, keyLen, data, size); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pHashObj->size += 1; + + // Too many records, needs to bucket split + if ((pHashObj->numOfBuckets * LHASH_CAP_RATIO * pHashObj->tuplesPerPage) < pHashObj->size) { + int32_t newBucketId = pHashObj->numOfBuckets; + + int32_t code = doAddNewBucket(pHashObj); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t numOfBits = ceil(log(pHashObj->numOfBuckets) / log(2)); + if (numOfBits > pHashObj->bits) { +// printf("extend the bits from %d to %d, new bucket:%d\n", pHashObj->bits, numOfBits, newBucketId); + ASSERT(numOfBits == pHashObj->bits + 1); + pHashObj->bits = numOfBits; + } + + int32_t splitBucketId = doGetRelatedSplitBucketId(newBucketId, pHashObj->bits); + + // load all data in this bucket and check if the data needs to relocated into the new bucket + SLHashBucket* pBucket = pHashObj->pBucket[splitBucketId]; +// printf("split %d items' bucket:0x%x to new bucket:0x%x\n", pBucket->size, splitBucketId, newBucketId); + + for (int32_t i = 0; i < taosArrayGetSize(pBucket->pPageIdList); ++i) { + int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i); + SFilePage* p = getBufPage(pHashObj->pBuf, pageId); + + char* pStart = p->data; + while (pStart - ((char*) p) < p->num) { + SLHashNode* pNode = (SLHashNode*)pStart; + ASSERT(pNode->keyLen > 0 && pNode->dataLen >= 0); + + char* k = GET_LHASH_NODE_KEY(pNode); + int32_t hashv = pHashObj->hashFn(k, pNode->keyLen); + int32_t v1 = doGetBucketIdFromHashVal(hashv, pHashObj->bits); + + if (v1 != splitBucketId) { // place it into the new bucket + ASSERT(v1 == newBucketId); +// printf("move key:%d to 0x%x bucket, remain items:%d\n", *(int32_t*)k, v1, pBucket->size - 1); + + SLHashBucket* pNewBucket = pHashObj->pBucket[newBucketId]; + doAddToBucket(pHashObj, pNewBucket, newBucketId, (void*)GET_LHASH_NODE_KEY(pNode), pNode->keyLen, + GET_LHASH_NODE_KEY(pNode), pNode->dataLen); + doRemoveFromBucket(p, pNode, pBucket); + } else { +// printf("check key:%d, located into: %d, skip it\n", *(int*) k, v1); + + int32_t nodeSize = GET_LHASH_NODE_LEN(pStart); + pStart += nodeSize; + } + } + releaseBufPage(pHashObj->pBuf, p); + } + + doCompressBucketPages(pHashObj, pBucket); + } + + return TSDB_CODE_SUCCESS; +} + +char* tHashGet(SLHashObj* pHashObj, const void *key, size_t keyLen) { + ASSERT(pHashObj != NULL && key != NULL && keyLen > 0); + int32_t hashv = pHashObj->hashFn(key, keyLen); + + int32_t bucketId = doGetBucketIdFromHashVal(hashv, pHashObj->bits); + if (bucketId >= pHashObj->numOfBuckets) { + bucketId = doGetAlternativeBucketId(bucketId, pHashObj->bits, pHashObj->numOfBuckets); + } + + SLHashBucket* pBucket = pHashObj->pBucket[bucketId]; + for (int32_t i = 0; i < taosArrayGetSize(pBucket->pPageIdList); ++i) { + int32_t pageId = *(int32_t*)taosArrayGet(pBucket->pPageIdList, i); + SFilePage* p = getBufPage(pHashObj->pBuf, pageId); + + char* pStart = p->data; + while (pStart - p->data < p->num) { + SLHashNode* pNode = (SLHashNode*)pStart; + + char* k = GET_LHASH_NODE_KEY(pNode); + if (pNode->keyLen == keyLen && (memcmp(key, k, keyLen) == 0)) { + releaseBufPage(pHashObj->pBuf, p); + return GET_LHASH_NODE_DATA(pNode); + } else { + pStart += GET_LHASH_NODE_LEN(pStart); + } + } + + releaseBufPage(pHashObj->pBuf, p); + } + + return NULL; +} + +int32_t tHashRemove(SLHashObj* pHashObj, const void *key, size_t keyLen) { + // todo +} + +void tHashPrint(const SLHashObj* pHashObj, int32_t type) { + printf("==================== linear hash ====================\n"); + printf("total bucket:%d, size:%ld, ratio:%.2f\n", pHashObj->numOfBuckets, pHashObj->size, LHASH_CAP_RATIO); + + dBufSetPrintInfo(pHashObj->pBuf); + + if (type == LINEAR_HASH_DATA) { + for (int32_t i = 0; i < pHashObj->numOfBuckets; ++i) { +// printf("bucket: 0x%x, obj:%d, page:%d\n", i, pHashObj->pBucket[i]->size, +// (int)taosArrayGetSize(pHashObj->pBucket[i]->pPageIdList)); + } + } else { + dBufPrintStatis(pHashObj->pBuf); + } +} \ No newline at end of file diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c new file mode 100644 index 0000000000000000000000000000000000000000..d042dc0effc536ac9124e9f2c2688e4c5715ab54 --- /dev/null +++ b/source/libs/executor/src/tsort.c @@ -0,0 +1,679 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tcommon.h" +#include "query.h" + +#include "tsort.h" +#include "tep.h" +#include "tdef.h" +#include "tlosertree.h" +#include "tpagedbuf.h" +#include "tutil.h" + +typedef struct STupleHandle { + SSDataBlock* pBlock; + int32_t rowIndex; +} STupleHandle; + +typedef struct SSortHandle { + int32_t type; + + int32_t pageSize; + int32_t numOfPages; + SDiskbasedBuf *pBuf; + + SArray *pOrderInfo; + bool nullFirst; + bool hasVarCol; + SArray *pOrderedSource; + + _sort_fetch_block_fn_t fetchfp; + _sort_merge_compar_fn_t comparFn; + + void *pParam; + SMultiwayMergeTreeInfo *pMergeTree; + int32_t numOfCols; + + int64_t startTs; + uint64_t sortElapsed; + uint64_t totalElapsed; + + int32_t sourceId; + SSDataBlock *pDataBlock; + SMsortComparParam cmpParam; + int32_t numOfCompletedSources; + bool opened; + const char *idStr; + + bool inMemSort; + bool needAdjust; + STupleHandle tupleHandle; +} SSortHandle; + +static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param); + +static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) { + SSDataBlock* pBlock = calloc(1, sizeof(SSDataBlock)); + pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + + colInfo.info.type = pSchema[i].type; + colInfo.info.bytes = pSchema[i].bytes; + colInfo.info.colId = pSchema[i].colId; + taosArrayPush(pBlock->pDataBlock, &colInfo); + } + + return pBlock; +} + +/** + * + * @param type + * @return + */ +SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) { + SSortHandle* pSortHandle = calloc(1, sizeof(SSortHandle)); + + pSortHandle->type = type; + pSortHandle->pageSize = pageSize; + pSortHandle->numOfPages = numOfPages; + pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); + pSortHandle->pOrderInfo = pOrderInfo; + pSortHandle->nullFirst = nullFirst; + pSortHandle->cmpParam.orderInfo = pOrderInfo; + + pSortHandle->pDataBlock = createDataBlock_rv(pSchema, numOfCols); + tsortSetComparFp(pSortHandle, msortComparFn); + + if (idstr != NULL) { + pSortHandle->idStr = strdup(idstr); + } + + return pSortHandle; +} + +void tsortDestroySortHandle(SSortHandle* pSortHandle) { + tsortClose(pSortHandle); + if (pSortHandle->pMergeTree != NULL) { + tMergeTreeDestroy(pSortHandle->pMergeTree); + } + + destroyDiskbasedBuf(pSortHandle->pBuf); + tfree(pSortHandle->idStr); + tfree(pSortHandle); +} + +int32_t tsortAddSource(SSortHandle* pSortHandle, void* pSource) { + taosArrayPush(pSortHandle->pOrderedSource, &pSource); +} + +static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSources, SSDataBlock* pBlock, int32_t* sourceId) { + SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + if (pSource == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + pSource->pageIdList = getDataBufPagesIdList(pBuf, (*sourceId)); + pSource->src.pBlock = pBlock; + + taosArrayPush(pAllSources, &pSource); + + (*sourceId) += 1; + + int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock))/rowSize; + + return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); +} + +static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { + int32_t start = 0; + + if (pHandle->pBuf == NULL) { + int32_t code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + while(start < pDataBlock->info.rows) { + int32_t stop = 0; + blockDataSplitRows(pDataBlock, pHandle->hasVarCol, start, &stop, pHandle->pageSize); + SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1); + if (p == NULL) { + return terrno; + } + + int32_t pageId = -1; + SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + if (pPage == NULL) { + return terrno; + } + + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage->data, p); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + blockDataDestroy(p); + start = stop + 1; + } + + blockDataClearup(pDataBlock, pHandle->hasVarCol); + + SSDataBlock* pBlock = createOneDataBlock(pDataBlock); + int32_t code = doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } +} + +static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SSortHandle* pHandle) { + cmpParam->pSources = taosArrayGet(pSources, startIndex); + cmpParam->numOfSources = (endIndex - startIndex + 1); + + int32_t code = 0; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pHandle->pBuf, pPage); + } + } else { + // multi-pass internal merge sort is required + if (pHandle->pBuf == NULL) { + code = createDiskbasedBuf(&pHandle->pBuf, pHandle->pageSize, pHandle->numOfPages * pHandle->pageSize, 0, "/tmp"); + dBufSetPrintInfo(pHandle->pBuf); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + for (int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SGenericSource* pSource = cmpParam->pSources[i]; + pSource->src.pBlock = pHandle->fetchfp(pSource->param); + } + } + + return code; +} + +static int32_t sortComparClearup(SMsortComparParam* cmpParam) { + for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + blockDataDestroy(pSource->src.pBlock); + tfree(pSource); + } + + cmpParam->numOfSources = 0; +} + +static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); + bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); + + if (isNull) { + colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + } else { + char* pData = colDataGetData(pSrcColInfo, *rowIndex); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } + } + + pBlock->info.rows += 1; + *rowIndex += 1; +} + +static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SSortHandle *pHandle, int32_t* numOfCompleted) { + /* + * load a new SDataBlock into memory of a given intermediate data-set source, + * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree + */ + if (pSource->src.rowIndex >= pSource->src.pBlock->info.rows) { + pSource->src.rowIndex = 0; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { + (*numOfCompleted) += 1; + pSource->src.rowIndex = -1; + pSource->pageIndex = -1; + pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); + } else { + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pHandle->pBuf, pPage); + } + } else { + pSource->src.pBlock = pHandle->fetchfp(((SGenericSource*)pSource)->param); + if (pSource->src.pBlock == NULL) { + (*numOfCompleted) += 1; + pSource->src.rowIndex = -1; + } + } + } + + /* + * Adjust loser tree otherwise, according to new candidate data + * if the loser tree is rebuild completed, we do not need to adjust + */ + int32_t leafNodeIndex = tMergeTreeGetAdjustIndex(pTree); + +#ifdef _DEBUG_VIEW + printf("before adjust:\t"); + tMergeTreePrint(pTree); +#endif + + tMergeTreeAdjust(pTree, leafNodeIndex); + +#ifdef _DEBUG_VIEW + printf("\nafter adjust:\t"); + tMergeTreePrint(pTree); +#endif +} + +static SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SMsortComparParam* cmpParam, int32_t capacity) { + blockDataClearup(pHandle->pDataBlock, pHandle->hasVarCol); + + while(1) { + if (cmpParam->numOfSources == pHandle->numOfCompletedSources) { + break; + } + + int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + + SExternalMemSource *pSource = (*cmpParam).pSources[index]; + appendOneRowToDataBlock(pHandle->pDataBlock, pSource->src.pBlock, &pSource->src.rowIndex); + + int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + + if (pHandle->pDataBlock->info.rows >= capacity) { + return pHandle->pDataBlock; + } + } + + return (pHandle->pDataBlock->info.rows > 0)? pHandle->pDataBlock:NULL; +} + +int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t pLeftIdx = *(int32_t *)pLeft; + int32_t pRightIdx = *(int32_t *)pRight; + + SMsortComparParam *pParam = (SMsortComparParam *)param; + + SArray *pInfo = pParam->orderInfo; + + SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; + SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; + + // this input is exhausted, set the special value to denote this + if (pLeftSource->src.rowIndex == -1) { + return 1; + } + + if (pRightSource->src.rowIndex == -1) { + return -1; + } + + SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; + SSDataBlock* pRightBlock = pRightSource->src.pBlock; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + } + + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pParam->nullFirst? 1:-1; + } + + if (leftNull) { + return pParam->nullFirst? -1:1; + } + + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + + switch(pLeftColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftv = *(int32_t*)left1; + int32_t rightv = *(int32_t*)right1; + + if (leftv == rightv) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return leftv < rightv? -1 : 1; + } else { + return leftv < rightv? 1 : -1; + } + } + } + default: + assert(0); + } + } +} + +static int32_t doInternalMergeSort(SSortHandle* pHandle) { + size_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + + // Calculate the I/O counts to complete the data sort. + double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages)); + + pHandle->totalElapsed = taosGetTimestampUs() - pHandle->startTs; + qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, + pHandle->idStr, (int32_t) (sortPass + 1), getTotalBufSize(pHandle->pBuf), pHandle->sortElapsed, pHandle->totalElapsed); + + size_t pgSize = pHandle->pageSize; + int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pHandle->pDataBlock))/ blockDataGetSerialRowSize(pHandle->pDataBlock); + + blockDataEnsureCapacity(pHandle->pDataBlock, numOfRows); + + size_t numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); + for(int32_t t = 0; t < sortPass; ++t) { + int64_t st = taosGetTimestampUs(); + + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + + int32_t numOfInputSources = pHandle->numOfPages; + int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources; + + // Only *numOfInputSources* can be loaded into buffer to perform the external sort. + for(int32_t i = 0; i < sortGroup; ++i) { + pHandle->sourceId += 1; + + int32_t end = (i + 1) * numOfInputSources - 1; + if (end > numOfSorted - 1) { + end = numOfSorted - 1; + } + + pHandle->cmpParam.numOfSources = end - i * numOfInputSources + 1; + + int32_t code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, i * numOfInputSources, end, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + while (1) { + SSDataBlock* pDataBlock = getSortedBlockData(pHandle, &pHandle->cmpParam, numOfRows); + if (pDataBlock == NULL) { + break; + } + + int32_t pageId = -1; + SFilePage* pPage = getNewBufPage(pHandle->pBuf, pHandle->sourceId, &pageId); + if (pPage == NULL) { + return terrno; + } + + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pHandle->pBuf)); + + blockDataToBuf(pPage->data, pDataBlock); + + setBufPageDirty(pPage, true); + releaseBufPage(pHandle->pBuf, pPage); + + blockDataClearup(pDataBlock, pHandle->hasVarCol); + } + + tMergeTreeDestroy(pHandle->pMergeTree); + pHandle->numOfCompletedSources = 0; + + SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock); + code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId); + if (code != 0) { + return code; + } + } + + sortComparClearup(&pHandle->cmpParam); + + taosArrayClear(pHandle->pOrderedSource); + taosArrayAddAll(pHandle->pOrderedSource, pResList); + taosArrayDestroy(pResList); + + numOfSorted = taosArrayGetSize(pHandle->pOrderedSource); + + int64_t el = taosGetTimestampUs() - st; + pHandle->totalElapsed += el; + + SDiskbasedBufStatis statis = getDBufStatis(pHandle->pBuf); + qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", pHandle->idStr, t + 1, el, statis.loadBytes/1024.0, + statis.flushBytes/1024.0); + + if (pHandle->type == SORT_MULTISOURCE_MERGE) { + pHandle->type = SORT_SINGLESOURCE_SORT; + pHandle->comparFn = msortComparFn; + } + } + + pHandle->cmpParam.numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + return 0; +} + +static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) { + size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize; + + if (pHandle->type == SORT_SINGLESOURCE_SORT) { + SGenericSource* source = taosArrayGetP(pHandle->pOrderedSource, 0); + taosArrayClear(pHandle->pOrderedSource); + + while (1) { + SSDataBlock* pBlock = pHandle->fetchfp(source->param); + if (pBlock == NULL) { + break; + } + + if (pHandle->pDataBlock == NULL) { + pHandle->pDataBlock = createOneDataBlock(pBlock); + } + + int32_t code = blockDataMerge(pHandle->pDataBlock, pBlock); + if (code != 0) { + return code; + } + + size_t size = blockDataGetSize(pHandle->pDataBlock); + if (size > sortBufSize) { + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst); + + int64_t el = taosGetTimestampUs() - p; + pHandle->sortElapsed += el; + + doAddToBuf(pHandle->pDataBlock, pHandle); + } + } + + if (pHandle->pDataBlock->info.rows > 0) { + size_t size = blockDataGetSize(pHandle->pDataBlock); + + // Perform the in-memory sort and then flush data in the buffer into disk. + blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst); + + // All sorted data can fit in memory, external memory sort is not needed. Return to directly + if (size <= sortBufSize) { + pHandle->cmpParam.numOfSources = 1; + pHandle->inMemSort = true; + + pHandle->tupleHandle.rowIndex = -1; + pHandle->tupleHandle.pBlock = pHandle->pDataBlock; + return 0; + } else { + doAddToBuf(pHandle->pDataBlock, pHandle); + } + } + + tfree(source); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tsortOpen(SSortHandle* pHandle) { + if (pHandle->opened) { + return 0; + } + + if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) { + return -1; + } + + pHandle->opened = true; + + int32_t code = createInitialSortedMultiSources(pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // do internal sort + code = doInternalMergeSort(pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + int32_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource); + if (pHandle->pBuf != NULL) { + ASSERT(numOfSources <= getNumOfInMemBufPages(pHandle->pBuf)); + } + + code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, 0, numOfSources - 1, pHandle); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = tMergeTreeCreate(&pHandle->pMergeTree, pHandle->cmpParam.numOfSources, &pHandle->cmpParam, pHandle->comparFn); + if (code != TSDB_CODE_SUCCESS) { + return code; + } +} + +int32_t tsortClose(SSortHandle* pHandle) { + // do nothing +} + +int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fp) { + pHandle->fetchfp = fp; +} + +int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { + pHandle->comparFn = fp; +} + +STupleHandle* tsortNextTuple(SSortHandle* pHandle) { + if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { + return NULL; + } + + // All the data are hold in the buffer, no external sort is invoked. + if (pHandle->inMemSort) { + pHandle->tupleHandle.rowIndex += 1; + if (pHandle->tupleHandle.rowIndex == pHandle->pDataBlock->info.rows) { + pHandle->numOfCompletedSources = 1; + return NULL; + } + + return &pHandle->tupleHandle; + } + + int32_t index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + SExternalMemSource *pSource = pHandle->cmpParam.pSources[index]; + + if (pHandle->needAdjust) { + int32_t code = adjustMergeTreeForNextTuple(pSource, pHandle->pMergeTree, pHandle, &pHandle->numOfCompletedSources); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } + } + + if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { + return NULL; + } + + // Get the adjusted value after the loser tree is updated. + index = tMergeTreeGetChosenIndex(pHandle->pMergeTree); + pSource = pHandle->cmpParam.pSources[index]; + + assert(pSource->src.pBlock != NULL); + + pHandle->tupleHandle.rowIndex = pSource->src.rowIndex; + pHandle->tupleHandle.pBlock = pSource->src.pBlock; + + pHandle->needAdjust = true; + pSource->src.rowIndex += 1; + + return &pHandle->tupleHandle; +} + +bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) { + return false; +} + +void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) { + SColumnInfoData* pColInfo = TARRAY_GET_ELEM(pVHandle->pBlock->pDataBlock, colIndex); + return colDataGetData(pColInfo, pVHandle->rowIndex); +} diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index c9657607c2ec406814836b030dafc02c77383bf2..262232ceb19b792f53af27e21b00dbfd202cd30d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #include #include @@ -37,22 +38,30 @@ namespace { +enum { + data_rand = 0x1, + data_asc = 0x2, + data_desc = 0x3, +}; + typedef struct SDummyInputInfo { - int32_t max; - int32_t current; - int32_t startVal; + int32_t totalPages; // numOfPages + int32_t current; + int32_t startVal; + int32_t type; + int32_t numOfRowsPerPage; + int32_t numOfCols; // number of columns + int64_t tsStart; SSDataBlock* pBlock; } SDummyInputInfo; SSDataBlock* getDummyBlock(void* param, bool* newgroup) { SOperatorInfo* pOperator = static_cast(param); SDummyInputInfo* pInfo = static_cast(pOperator->info); - if (pInfo->current >= pInfo->max) { + if (pInfo->current >= pInfo->totalPages) { return NULL; } - int32_t numOfRows = 1000; - if (pInfo->pBlock == NULL) { pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); @@ -62,8 +71,8 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { colInfo.info.type = TSDB_DATA_TYPE_INT; colInfo.info.bytes = sizeof(int32_t); colInfo.info.colId = 1; - colInfo.pData = static_cast(calloc(numOfRows, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(calloc(1, (numOfRows + 7) / 8)); + colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); @@ -85,10 +94,18 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { char buf[128] = {0}; char b1[128] = {0}; - for(int32_t i = 0; i < numOfRows; ++i) { + int32_t v = 0; + for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); - int32_t v = (--pInfo->startVal); + if (pInfo->type == data_desc) { + v = (--pInfo->startVal); + } else if (pInfo->type == data_asc) { + v = ++pInfo->startVal; + } else if (pInfo->type == data_rand) { + v = random(); + } + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); // sprintf(buf, "this is %d row", i); @@ -98,21 +115,103 @@ SSDataBlock* getDummyBlock(void* param, bool* newgroup) { // colDataAppend(pColInfo2, i, b1, false); } - pBlock->info.rows = numOfRows; + pBlock->info.rows = pInfo->numOfRowsPerPage; pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; } -SOperatorInfo* createDummyOperator(int32_t numOfBlocks) { +SSDataBlock* get2ColsDummyBlock(void* param, bool* newgroup) { + SOperatorInfo* pOperator = static_cast(param); + SDummyInputInfo* pInfo = static_cast(pOperator->info); + if (pInfo->current >= pInfo->totalPages) { + return NULL; + } + + if (pInfo->pBlock == NULL) { + pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + + pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; + colInfo.info.bytes = sizeof(int64_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); +// colInfo.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + + SColumnInfoData colInfo1 = {0}; + colInfo1.info.type = TSDB_DATA_TYPE_INT; + colInfo1.info.bytes = 4; + colInfo1.info.colId = 2; + + colInfo1.pData = static_cast(calloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); + colInfo1.nullbitmap = static_cast(calloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + } else { + blockDataClearup(pInfo->pBlock, false); + } + + SSDataBlock* pBlock = pInfo->pBlock; + + char buf[128] = {0}; + char b1[128] = {0}; + int64_t ts = 0; + int32_t v = 0; + for(int32_t i = 0; i < pInfo->numOfRowsPerPage; ++i) { + SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); + + ts = (++pInfo->tsStart); + colDataAppend(pColInfo, i, reinterpret_cast(&ts), false); + + SColumnInfoData* pColInfo1 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); + if (pInfo->type == data_desc) { + v = (--pInfo->startVal); + } else if (pInfo->type == data_asc) { + v = ++pInfo->startVal; + } else if (pInfo->type == data_rand) { + v = random(); + } + + colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); + +// sprintf(buf, "this is %d row", i); +// STR_TO_VARSTR(b1, buf); +// +// SColumnInfoData* pColInfo2 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); +// colDataAppend(pColInfo2, i, b1, false); + } + + pBlock->info.rows = pInfo->numOfRowsPerPage; + pBlock->info.numOfCols = 1; + + pInfo->current += 1; + + blockDataUpdateTsWindow(pBlock); + return pBlock; + +} + +SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_t rowsPerPage, int32_t type, int32_t numOfCols) { SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); pOperator->name = "dummyInputOpertor4Test"; - pOperator->exec = getDummyBlock; + + if (numOfCols == 1) { + pOperator->exec = getDummyBlock; + } else { + pOperator->exec = get2ColsDummyBlock; + } SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); - pInfo->max = numOfBlocks; - pInfo->startVal = 1500000; + pInfo->totalPages = numOfBlocks; + pInfo->startVal = startVal; + pInfo->numOfRowsPerPage = rowsPerPage; + pInfo->type = type; + pInfo->tsStart = 1620000000000; pOperator->info = pInfo; return pOperator; @@ -123,6 +222,7 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } +#if 0 TEST(testCase, build_executor_tree_Test) { const char* msg = "{\n" "\t\"Id\":\t{\n" @@ -218,34 +318,34 @@ TEST(testCase, build_executor_tree_Test) { // int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } -//TEST(testCase, inMem_sort_Test) { -// SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); -// SOrder o = {.order = TSDB_ORDER_ASC}; -// o.col.info.colId = 1; -// o.col.info.type = TSDB_DATA_TYPE_INT; -// taosArrayPush(pOrderVal, &o); -// -// SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); -// SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); -// exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); -// taosArrayPush(pExprInfo, &exp); -// -// SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); -// exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); -// taosArrayPush(pExprInfo, &exp1); -// -// SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal); -// -// bool newgroup = false; -// SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup); -// -// SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); -// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); -// for(int32_t i = 0; i < pRes->info.rows; ++i) { -// char* p = colDataGet(pCol2, i); -// printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); -// } -//} +TEST(testCase, inMem_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); + taosArrayPush(pExprInfo, &exp1); + + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL); + + bool newgroup = false; + SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup); + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); + SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for(int32_t i = 0; i < pRes->info.rows; ++i) { + char* p = colDataGetData(pCol2, i); + printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } +} typedef struct su { int32_t v; @@ -300,7 +400,7 @@ TEST(testCase, external_sort_Test) { exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); // taosArrayPush(pExprInfo, &exp1); - SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal); + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL); bool newgroup = false; SSDataBlock* pRes = NULL; @@ -326,13 +426,159 @@ TEST(testCase, external_sort_Test) { SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); // SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); for (int32_t i = 0; i < pRes->info.rows; ++i) { -// char* p = colDataGet(pCol2, i); +// char* p = colDataGetData(pCol2, i); printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]); // printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); } } - printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf); + int64_t s2 = taosGetTimestampUs(); + printf("total:%ld\n", s2 - s1); + + pOperator->cleanupFn(pOperator->info, 2); + tfree(exp); + tfree(exp1); + taosArrayDestroy(pExprInfo); + taosArrayDestroy(pOrderVal); +} + +TEST(testCase, sorted_merge_Test) { + srand(time(NULL)); + + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 1, "count_result"); + exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns->flag = TSDB_COL_NORMAL; + exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; + exp->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); +// taosArrayPush(pExprInfo, &exp1); + + int32_t numOfSources = 10; + SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*)); + for(int32_t i = 0; i < numOfSources; ++i) { + plist[i] = createDummyOperator(1, 1, 1, data_asc); + } + + SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL); + + bool newgroup = false; + SSDataBlock* pRes = NULL; + + int32_t total = 1; + + int64_t s1 = taosGetTimestampUs(); + int32_t t = 1; + + while(1) { + int64_t s = taosGetTimestampUs(); + pRes = pOperator->exec(pOperator, &newgroup); + + int64_t e = taosGetTimestampUs(); + if (t++ == 1) { + printf("---------------elapsed:%ld\n", e - s); + } + + if (pRes == NULL) { + break; + } + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for (int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGetData(pCol2, i); + printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); +// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } + } + + int64_t s2 = taosGetTimestampUs(); + printf("total:%ld\n", s2 - s1); + + pOperator->cleanupFn(pOperator->info, 2); + tfree(exp); + tfree(exp1); + taosArrayDestroy(pExprInfo); + taosArrayDestroy(pOrderVal); +} + +#endif + +TEST(testCase, time_interval_Operator_Test) { + srand(time(NULL)); + + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1, "ts"); + exp->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp->base.pColumns->flag = TSDB_COL_NORMAL; + exp->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8}; + exp->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BIGINT, 8, 2, "res1"); + exp1->base.pColumns = static_cast(calloc(1, sizeof(SColumn))); + exp1->base.pColumns->flag = TSDB_COL_NORMAL; + exp1->base.pColumns->info = (SColumnInfo) {.colId = 1, .type = TSDB_DATA_TYPE_INT, .bytes = 4}; + exp1->base.numOfCols = 1; + + taosArrayPush(pExprInfo, &exp1); + + SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2); + + SExecTaskInfo ti = {0}; + SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti); + + bool newgroup = false; + SSDataBlock* pRes = NULL; + + int32_t total = 1; + + int64_t s1 = taosGetTimestampUs(); + int32_t t = 1; + + while(1) { + int64_t s = taosGetTimestampUs(); + pRes = pOperator->exec(pOperator, &newgroup); + + int64_t e = taosGetTimestampUs(); + if (t++ == 1) { + printf("---------------elapsed:%ld\n", e - s); + } + + if (pRes == NULL) { + break; + } + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for (int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGetData(pCol2, i); + printf("%d: %ld\n", total++, ((int64_t*)pCol1->pData)[i]); +// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } + } int64_t s2 = taosGetTimestampUs(); printf("total:%ld\n", s2 - s1); @@ -343,4 +589,5 @@ TEST(testCase, external_sort_Test) { taosArrayDestroy(pExprInfo); taosArrayDestroy(pOrderVal); } + #pragma GCC diagnostic pop diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..66ef3b0877d67f3967a1b642e03d84c4e979c8d5 --- /dev/null +++ b/source/libs/executor/test/lhashTests.cpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include "executorimpl.h" +#include "tlinearhash.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +TEST(testCase, linear_hash_Tests) { + srand(time(NULL)); + + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); +#if 0 + SLHashObj* pHashObj = tHashInit(256, 4096, fn, 320); + for(int32_t i = 0; i < 5000000; ++i) { + int32_t code = tHashPut(pHashObj, &i, sizeof(i), &i, sizeof(i)); + assert(code == 0); + } + +// tHashPrint(pHashObj, LINEAR_HASH_STATIS); + +// for(int32_t i = 0; i < 10000; ++i) { +// char* v = tHashGet(pHashObj, &i, sizeof(i)); +// if (v != NULL) { +//// printf("find value: %d, key:%d\n", *(int32_t*) v, i); +// } else { +// printf("failed to found key:%d in hash\n", i); +// } +// } + + tHashPrint(pHashObj, LINEAR_HASH_STATIS); + tHashCleanup(pHashObj); +#endif + +#if 0 + SHashObj* pHashObj = taosHashInit(1000, fn, false, HASH_NO_LOCK); + for(int32_t i = 0; i < 1000000; ++i) { + taosHashPut(pHashObj, &i, sizeof(i), &i, sizeof(i)); + } + + for(int32_t i = 0; i < 10000; ++i) { + void* v = taosHashGet(pHashObj, &i, sizeof(i)); + } + taosHashCleanup(pHashObj); +#endif + +} \ No newline at end of file diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fc366e4cc872ca365c36b8724e7cb40e0ee29a40 --- /dev/null +++ b/source/libs/executor/test/sortTests.cpp @@ -0,0 +1,280 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "executor.h" +#include "stub.h" +#include "taos.h" +#include "tdef.h" +#include "tep.h" +#include "trpc.h" +#include "tvariant.h" + +namespace { +typedef struct { + int32_t startVal; + int32_t count; + int32_t pageRows; +} _info; + +SSDataBlock* getSingleColDummyBlock(void* param) { + _info* pInfo = (_info*) param; + if (--pInfo->count < 0) { + return NULL; + } + + SSDataBlock* pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(pInfo->pageRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (pInfo->pageRows + 7) / 8)); + + taosArrayPush(pBlock->pDataBlock, &colInfo); + + for (int32_t i = 0; i < pInfo->pageRows; ++i) { + SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); + + int32_t v = ++pInfo->startVal; + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + } + + pBlock->info.rows = pInfo->pageRows; + pBlock->info.numOfCols = 1; + return pBlock; +} + +int32_t docomp(const void* p1, const void* p2, void* param) { + int32_t pLeftIdx = *(int32_t *)p1; + int32_t pRightIdx = *(int32_t *)p2; + + SMsortComparParam *pParam = (SMsortComparParam *)param; + SGenericSource** px = reinterpret_cast(pParam->pSources); + + SArray *pInfo = pParam->orderInfo; + + SGenericSource* pLeftSource = px[pLeftIdx]; + SGenericSource* pRightSource = px[pRightIdx]; + + // this input is exhausted, set the special value to denote this + if (pLeftSource->src.rowIndex == -1) { + return 1; + } + + if (pRightSource->src.rowIndex == -1) { + return -1; + } + + SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; + SSDataBlock* pRightBlock = pRightSource->src.pBlock; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = (SBlockOrderInfo*)TARRAY_GET_ELEM(pInfo, i); + + SColumnInfoData* pLeftColInfoData = (SColumnInfoData*)TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg); + } + + SColumnInfoData* pRightColInfoData = (SColumnInfoData*) TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg); + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pParam->nullFirst? 1:-1; + } + + if (leftNull) { + return pParam->nullFirst? -1:1; + } + + void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); + void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); + + switch(pLeftColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftv = *(int32_t*)left1; + int32_t rightv = *(int32_t*)right1; + + if (leftv == rightv) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return leftv < rightv? -1 : 1; + } else { + return leftv < rightv? 1 : -1; + } + } + } + default: + assert(0); + } + } + + return 0; +} +} // namespace + +#if 0 +TEST(testCase, inMem_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + int32_t numOfRows = 1000; + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortAddSource(phandle, &numOfRows); + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +TEST(testCase, external_mem_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4, }; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_SINGLESOURCE_SORT, 1024, 5, &s, 1, "test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + + _info* pInfo = (_info*) calloc(1, sizeof(_info)); + pInfo->startVal = 100000; + pInfo->pageRows = 1000; + pInfo->count = 50; + + SGenericSource* ps = static_cast(calloc(1, sizeof(SGenericSource))); + ps->param = pInfo; + + tsortAddSource(phandle, ps); + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +TEST(testCase, ordered_merge_sort_Test) { + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {.order = TSDB_ORDER_ASC}; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + int32_t numOfRows = 1000; + SBlockOrderInfo oi = {0}; + oi.order = TSDB_ORDER_ASC; + oi.colIndex = 0; + SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); + taosArrayPush(orderInfo, &oi); + + SSchema s = {.type = TSDB_DATA_TYPE_INT, .colId = 1, .bytes = 4}; + SSortHandle* phandle = tsortCreateSortHandle(orderInfo, false, SORT_MULTISOURCE_MERGE, 1024, 5, &s, 1,"test_abc"); + tsortSetFetchRawDataFp(phandle, getSingleColDummyBlock); + tsortSetComparFp(phandle, docomp); + + for(int32_t i = 0; i < 10; ++i) { + SGenericSource* p = static_cast(calloc(1, sizeof(SGenericSource))); + _info* c = static_cast<_info*>(calloc(1, sizeof(_info))); + c->count = 1; + c->pageRows = 1000; + c->startVal = 0; + + p->param = c; + tsortAddSource(phandle, p); + } + + int32_t code = tsortOpen(phandle); + int32_t row = 1; + + while(1) { + STupleHandle* pTupleHandle = tsortNextTuple(phandle); + if (pTupleHandle == NULL) { + break; + } + + void* v = tsortGetValue(pTupleHandle, 0); + printf("%d: %d\n", row++, *(int32_t*) v); + + } + tsortDestroySortHandle(phandle); +} + +#endif + +#pragma GCC diagnostic pop diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 73ce67bd2847b3f1b20769e1626514ad5a3646ee..d9eeb6eeeb324ea7b9c03d432a5eee7552c43ce7 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -29,6 +29,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = NULL, .finalizeFunc = NULL }, + { + .name = "sum", + .type = FUNCTION_TYPE_SUM, + .classification = FUNC_MGT_AGG_FUNC, + .checkFunc = stubCheckAndGetResultType, + .getEnvFunc = NULL, + .initFunc = NULL, + .processFunc = NULL, + .finalizeFunc = NULL + }, { .name = "concat", .type = FUNCTION_TYPE_CONCAT, diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 1aa5871468ad7db0933564763b93c7aedfce63a6..c36fc9c659b11c893b577f17432bdfc881108db6 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -29,9 +29,10 @@ #include "tcompression.h" //#include "queryLog.h" #include "tudf.h" +#include "tep.h" #define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput)) -#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes) +#define GET_INPUT_DATA(x, y) ((char*) colDataGetData((x)->pInput, (y))) #define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) #define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) @@ -3818,7 +3819,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { skey = ekey; } } - assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); +// assignVal(pCtx->pOutput, pCtx->pInput, pCtx->resDataInfo.bytes, pCtx->inputType); } else if (type == TSDB_FILL_NEXT) { TSKEY ekey = skey; char* val = NULL; @@ -3960,14 +3961,14 @@ static void ts_comp_finalize(SqlFunctionCtx *pCtx) { // qDebug("total timestamp :%"PRId64, pTSbuf->numOfTotal); // TODO refactor transfer ownership of current file - *(FILE **)pCtx->pOutput = pTSbuf->f; + *(TdFilePtr *)pCtx->pOutput = pTSbuf->pFile; pResInfo->complete = true; // get the file size - struct stat fStat; - if ((fstat(fileno(pTSbuf->f), &fStat) == 0)) { - pResInfo->numOfRes = fStat.st_size; + int64_t file_size; + if (taosFStatFile(pTSbuf->pFile, &file_size, NULL) == 0) { + pResInfo->numOfRes = (uint32_t )file_size; } pTSbuf->remainOpen = true; @@ -4395,7 +4396,7 @@ SFunctionFpSet fpSet[1] = { .addInput = count_function, .finalize = doFinalizer, .combine = count_func_merge, - } + }, }; SAggFunctionInfo aggFunc[35] = {{ diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index 7d9cb97400732c57bbb52c1abe5a85b47d0d057d..bede8b80fdeb738792eeb3f1685d774287099e3d 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -16,7 +16,7 @@ #include "function.h" #include "os.h" -#include "exception.h" +#include "texception.h" #include "taosdef.h" #include "tmsg.h" #include "tarray.h" diff --git a/source/libs/function/src/tfill.c b/source/libs/function/src/tfill.c index db1bb3e7b8dd68f71c6e25bd7de52693d1b3f2d0..653e30cec883b1bb82a635e945cf1ec3077fcfba 100644 --- a/source/libs/function/src/tfill.c +++ b/source/libs/function/src/tfill.c @@ -23,7 +23,7 @@ #include "tfill.h" #include "thash.h" #include "function.h" -#include "common.h" +#include "tcommon.h" #include "ttime.h" #define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC) diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 40731adc58326da548bd53d4de6f2825efeff021..1aa0c04ec4f28583dcef72e087c888c7599518af 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -12,8 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include -#include "os.h" + +#include "tglobal.h" +#include "tcompare.h" #include "taosdef.h" #include "tcompare.h" @@ -221,7 +222,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, } pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT; - pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page + pBucket->bufPageSize = 16384 * 4; // 16k per page pBucket->type = dataType; pBucket->bytes = nElemSize; @@ -254,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir); + int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp"); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; @@ -269,7 +270,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { return; } - destroyResultBuf(pBucket->pBuffer); + destroyDiskbasedBuf(pBucket->pBuffer); tfree(pBucket->pSlots); tfree(pBucket); } @@ -347,7 +348,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { pSlot->info.data = NULL; } - pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId); + pSlot->info.data = getNewBufPage(pBucket->pBuffer, groupId, &pageId); pSlot->info.pageId = pageId; } diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 90ad1e15f4b2669620253e52269003abb0e4d1c7..59ed5b79f517f21f365831c6f175ee23842fafdc 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -16,12 +16,15 @@ #ifndef _TD_INDEX_INT_H_ #define _TD_INDEX_INT_H_ +#include "os.h" + #include "index.h" -#include "index_fst.h" #include "taos.h" +#include "tarray.h" #include "tchecksum.h" #include "thash.h" #include "tlog.h" +#include "tutil.h" #ifdef USE_LUCENE #include diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h index 9db913debfc06f3406aa4d9105709419e5314f1e..1445a1bc562cde1cf51fa3e6cb5df057eaa3cc88 100644 --- a/source/libs/index/inc/index_cache.h +++ b/source/libs/index/inc/index_cache.h @@ -15,10 +15,9 @@ #ifndef __INDEX_CACHE_H__ #define __INDEX_CACHE_H__ -#include "index.h" #include "indexInt.h" -#include "tlockfree.h" #include "tskiplist.h" + // ----------------- key structure in skiplist --------------------- /* A data row, the format is like below diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 072ec93e4e648ae55c945f291e0c0d73dc51cbc0..5da0dc537bfbefbe79a430d0220cef3149eb7f7b 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -20,11 +20,12 @@ extern "C" { #endif +#include "indexInt.h" +#include "index_fst_node.h" #include "index_fst_automation.h" #include "index_fst_counting_writer.h" #include "index_fst_registry.h" #include "index_fst_util.h" -#include "tarray.h" #define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a) diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index be056f38fa183e1460dac83c9fd90b70e69c40e9..786d3eb7d29b404dcbb1a20e4871c8661983e688 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -19,6 +19,7 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_util.h" typedef struct AutomationCtx AutomationCtx; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 1e0a88e17fc456a3b57ae874ed28711e2c2137a8..f8a62467239f55e8461fef562d25ee3cacaf20c1 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -16,12 +16,12 @@ #ifndef __INDEX_FST_COUNTING_WRITER_H__ #define __INDEX_FST_COUNTING_WRITER_H__ +#include "indexInt.h" + #ifdef __cplusplus extern "C" { #endif -#include "tfile.h" - //#define USE_MMAP 1 #define DefaultMem 1024 * 1024 @@ -38,7 +38,7 @@ typedef struct WriterCtx { WriterType type; union { struct { - int fd; + TdFilePtr pFile; bool readOnly; char buf[256]; int size; diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 32de7ea6211d4ab449fc6ef0d0fe8355ff380299..4dbf4cea04dcccc44b96d99709f3da287b05e1dd 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_counting_writer.h" #include "index_fst_util.h" diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index f94a21d8dd7fd6887f531793af74f2d5023f66b7..20b70e2866615a07c457fbfaa1919bbe1f17ef6c 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -19,9 +19,9 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_node.h" #include "index_fst_util.h" -#include "tarray.h" typedef struct FstRegistryCell { CompiledAddr addr; diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index 71a378584d401e966d7b2966ff215916c40c8232..f173dd9a15553b9a1997ff5915fc1e50ea9658a7 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -20,8 +20,8 @@ extern "C" { #endif +#include "indexInt.h" #include "index_fst_common.h" -#include "tarray.h" typedef uint64_t FstType; typedef uint64_t CompiledAddr; diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index 4618a39197d1d87b1e9d50f895578e9d2dd8fb85..f676651e52262d80cd0c7c2f42ec0fa6c258e43b 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -15,7 +15,6 @@ #ifndef __INDEX_TFILE_H__ #define __INDEX_TFILE_H__ -#include "index.h" #include "indexInt.h" #include "index_fst.h" #include "index_fst_counting_writer.h" diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h index adeb52bb8c989332ce8284d24d2369685c147323..985dd657d088c08b36ffac1f2f169420fe8e76b7 100644 --- a/source/libs/index/inc/index_util.h +++ b/source/libs/index/inc/index_util.h @@ -15,36 +15,46 @@ #ifndef __INDEX_UTIL_H__ #define __INDEX_UTIL_H__ +#include "indexInt.h" + #ifdef __cplusplus extern "C" { #endif -#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ - do { \ - memcpy((void*)buf, (void*)(&key->mem), sizeof(key->mem)); \ - buf += sizeof(key->mem); \ +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ } while (0) #define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ do { \ - memcpy((void*)buf, (void*)key->mem, len); \ + memcpy((void *)buf, (void *)key->mem, len); \ buf += len; \ } while (0) -#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ - do { \ - type c = var; \ - assert(sizeof(type) == sizeof(c)); \ - memcpy((void*)buf, (void*)&c, sizeof(c)); \ - buf += sizeof(c); \ +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(type) == sizeof(c)); \ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ } while (0) #define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ do { \ - memcpy((void*)buf, (void*)var, len); \ + memcpy((void *)buf, (void *)var, len); \ buf += len; \ } while (0) +/* multi sorted result intersection + * input: [1, 2, 4, 5] + * [2, 3, 4, 5] + * [1, 4, 5] + * output:[4, 5] + */ +void iIntersection(SArray *interResults, SArray *finalResult); +void iUnion(SArray *interResults, SArray *finalResult); #ifdef __cplusplus } #endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 9287a918285df77473b4bd2ae8b1c54fb79d00f8..5147734a85a47c2ba0a19f947920a8264750e868 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -370,22 +370,23 @@ static void indexInterResultsDestroy(SArray* results) { } taosArrayDestroy(results); } + static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* fResults) { // refactor, merge interResults into fResults by oType - SArray* first = taosArrayGetP(interResults, 0); - taosArraySort(first, uidCompare); - taosArrayRemoveDuplicate(first, uidCompare, NULL); + + for (int i = 0; i < taosArrayGetSize(interResults); i--) { + SArray* t = taosArrayGetP(interResults, i); + taosArraySort(t, uidCompare); + taosArrayRemoveDuplicate(t, uidCompare, NULL); + } if (oType == MUST) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); + iIntersection(interResults, fResults); } else if (oType == SHOULD) { - // just one column index, enhance later - taosArrayAddAll(fResults, first); - // tag1 condistion || tag2 condition + iUnion(interResults, fResults); } else if (oType == NOT) { // just one column index, enhance later - taosArrayAddAll(fResults, first); + taosArrayAddAll(fResults, interResults); // not use currently } return 0; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 3664bcfad0dc492a2383673f346b5605f5c02637..a6cabbd43940c91fb62afe9c65fc9dc4bb4139c3 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -235,6 +235,7 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran FstState st = fstStateCreate(OneTrans); fstStateSetCommInput(&st, trn->inp); + bool null = false; uint8_t inp = fstStateCommInput(&st, &null); if (null == true) { @@ -936,6 +937,7 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } + void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { FstLastTransition* trn = unNode->last; if (trn == NULL) { diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index 590ff294bf8f1841fc27e4519df1d23f668adcd5..ed1ad7a374286c8251dcb719be9a5510c6eec844 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -16,24 +16,24 @@ #include "index_fst_automation.h" StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { - StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); - if (nsv == NULL) { + StartWithStateValue* sv = calloc(1, sizeof(StartWithStateValue)); + if (sv == NULL) { return NULL; } - nsv->kind = kind; - nsv->type = ty; + sv->kind = kind; + sv->type = ty; if (ty == FST_INT) { - nsv->val = *(int*)val; + sv->val = *(int*)val; } else if (ty == FST_CHAR) { size_t len = strlen((char*)val); - nsv->ptr = (char*)calloc(1, len + 1); - memcpy(nsv->ptr, val, len); + sv->ptr = (char*)calloc(1, len + 1); + memcpy(sv->ptr, val, len); } else if (ty == FST_ARRAY) { // TODO, // nsv->arr = taosArrayFromList() } - return nsv; + return sv; } void startWithStateValueDestroy(void* val) { StartWithStateValue* sv = (StartWithStateValue*)val; @@ -146,11 +146,9 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (atype == AUTOMATION_ALWAYS) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTOMATION_PREFIX) { int val = 0; sv = startWithStateValueCreate(Running, FST_INT, &val); - ctx->stdata = (void*)sv; } else if (atype == AUTMMATION_MATCH) { } else { // add more search type @@ -160,9 +158,8 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) { if (data != NULL) { char* src = (char*)data; size_t len = strlen(src); - dst = (char*)malloc(len * sizeof(char) + 1); + dst = (char*)calloc(1, len * sizeof(char) + 1); memcpy(dst, src, len); - dst[len] = 0; } ctx->data = dst; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index b57f639726c5f738d2ba9d69b3aab74cb6f02dbf..16d98937780d39e13e683b00818acd7f660d0c3f 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#include "os.h" #include "index_fst_counting_writer.h" #include "indexInt.h" #include "index_fst_util.h" @@ -19,7 +21,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFile) { - assert(len == tfWrite(ctx->file.fd, buf, len)); + assert(len == taosWriteFile(ctx->file.pFile, buf, len)); } else { memcpy(ctx->mem.buf + ctx->offset, buf, len); } @@ -33,7 +35,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { nRead = len < ctx->file.size ? len : ctx->file.size; memcpy(buf, ctx->file.ptr, nRead); #else - nRead = tfRead(ctx->file.fd, buf, len); + nRead = taosReadFile(ctx->file.pFile, buf, len); #endif } else { memcpy(buf, ctx->mem.buf + ctx->offset, len); @@ -45,13 +47,13 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) { static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) { int nRead = 0; if (ctx->type == TFile) { - // tfLseek(ctx->file.fd, offset, 0); + // tfLseek(ctx->file.pFile, offset, 0); #ifdef USE_MMAP int32_t last = ctx->file.size - offset; nRead = last >= len ? len : last; memcpy(buf, ctx->file.ptr + offset, nRead); #else - nRead = tfPread(ctx->file.fd, buf, len, offset); + nRead = taosPReadFile(ctx->file.pFile, buf, len, offset); #endif } else { // refactor later @@ -69,9 +71,9 @@ static int writeCtxGetSize(WriterCtx* ctx) { } static int writeCtxDoFlush(WriterCtx* ctx) { if (ctx->type == TFile) { - // taosFsyncFile(ctx->file.fd); - tfFsync(ctx->file.fd); - // tfFlush(ctx->file.fd); + // taosFsyncFile(ctx->file.pFile); + taosFsyncFile(ctx->file.pFile); + // tfFlush(ctx->file.pFile); } else { // do nothing } @@ -87,25 +89,25 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int // ugly code, refactor later ctx->file.readOnly = readOnly; if (readOnly == false) { - // ctx->file.fd = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenCreateWriteAppend(path); - tfFtruncate(ctx->file.fd, 0); - struct stat fstat; - stat(path, &fstat); - ctx->file.size = fstat.st_size; + // ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + taosFtruncateFile(ctx->file.pFile, 0); + int64_t file_size; + taosStatFile(path, &file_size, NULL); + ctx->file.size = (int)file_size; } else { - // ctx->file.fd = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); - ctx->file.fd = tfOpenRead(path); + // ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); + ctx->file.pFile = taosOpenFile(path, TD_FILE_READ); struct stat fstat; stat(path, &fstat); ctx->file.size = fstat.st_size; #ifdef USE_MMAP - ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.fd, ctx->file.size); + ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size); #endif } memcpy(ctx->file.buf, path, strlen(path)); - if (ctx->file.fd < 0) { + if (ctx->file.pFile == NULL) { indexError("failed to open file, error %d", errno); goto END; } @@ -133,7 +135,7 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) { free(ctx->mem.buf); } else { ctx->flush(ctx); - tfClose(ctx->file.fd); + taosCloseFile(&ctx->file.pFile); if (ctx->file.readOnly) { #ifdef USE_MMAP munmap(ctx->file.ptr, ctx->file.size); diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index 07afc52ecf12bf515e32afca7fc50a44e0a422b3..b28b518fc156b5d7585a2e071b603a186bcd54e7 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "index_fst_registry.h" uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index aff976e52bb46836d42c1910cd913e96386afcb8..e44f8fc1c38b2eec23d3c22eb9efadc67fac565c 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -371,7 +371,7 @@ int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { return ret; } - IndexTFile* pTfile = (IndexTFile*)tfile; + IndexTFile* pTfile = tfile; SIndexTerm* term = query->term; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; @@ -586,11 +586,11 @@ static int tfileReaderLoadHeader(TFileReader* reader) { int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0); if (nread == -1) { - indexError("actual Read: %d, to read: %d, errno: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - errno, reader->ctx->file.fd, reader->ctx->file.buf); + indexError("actual Read: %d, to read: %d, errno: %d, filename: %s", (int)(nread), (int)sizeof(buf), + errno, reader->ctx->file.buf); } else { - indexInfo("actual Read: %d, to read: %d, filefd: %d, filename: %s", (int)(nread), (int)sizeof(buf), - reader->ctx->file.fd, reader->ctx->file.buf); + indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), + reader->ctx->file.buf); } // assert(nread == sizeof(buf)); memcpy(&reader->header, buf, sizeof(buf)); diff --git a/source/libs/index/src/index_util.c b/source/libs/index/src/index_util.c new file mode 100644 index 0000000000000000000000000000000000000000..fcaab968c24a296431f2a25039050947887e0aac --- /dev/null +++ b/source/libs/index/src/index_util.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "index_util.h" +#include "index.h" +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t *)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void iIntersection(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray *base = taosArrayGetP(inters, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t *)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(inters); j++) { + SArray *oth = taosArrayGetP(inters, j); + int mid = iBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t *)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(final, &tgt); + } + } + tfree(mi); +} +void iUnion(SArray *inters, SArray *final) { + int32_t sz = taosArrayGetSize(inters); + if (sz <= 0) { + return; + } + if (sz == 1) { + taosArrayAddAll(final, taosArrayGetP(inters, 0)); + return; + } + + MergeIndex *mi = calloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray *t = taosArrayGetP(inters, i); + mi[i].len = taosArrayGetSize(t); + mi[i].idx = 0; + } + while (1) { + uint64_t mVal = UINT_MAX; + int mIdx = -1; + + for (int j = 0; j < sz; j++) { + SArray *t = taosArrayGetP(inters, j); + if (mi[j].idx >= mi[j].len) { + continue; + } + uint64_t cVal = *(uint64_t *)taosArrayGet(t, mi[j].idx); + if (cVal < mVal) { + mVal = cVal; + mIdx = j; + } + } + if (mIdx != -1) { + mi[mIdx].idx++; + if (taosArrayGetSize(final) > 0) { + uint64_t lVal = *(uint64_t *)taosArrayGetLast(final); + if (lVal == mVal) { + continue; + } + } + taosArrayPush(final, &mVal); + } else { + break; + } + } + + tfree(mi); +} diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index 665dfd7318e0636d446365b42ae9c1d78de06f76..bed42be3e58af3277c8f6f5aada5b7673d63fe7d 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,6 +1,7 @@ add_executable(indexTest "") add_executable(fstTest "") add_executable(fstUT "") +add_executable(UtilUT "") target_sources(indexTest PRIVATE @@ -15,6 +16,11 @@ target_sources(fstUT PRIVATE "fstUT.cc" ) +target_sources(UtilUT + PRIVATE + "utilUT.cc" +) + target_include_directories ( indexTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" @@ -31,6 +37,12 @@ target_include_directories ( fstUT "${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +target_include_directories ( UtilUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries (indexTest os util @@ -53,6 +65,14 @@ target_link_libraries (fstUT index ) +target_link_libraries (UtilUT + os + util + common + gtest_main + index +) + #add_test( # NAME index_test diff --git a/source/libs/index/test/fstTest.cc b/source/libs/index/test/fstTest.cc index 65118a2bce043dc0e8725f0569a51e19c9bdebdc..698945ae14578d72377c3d30d942da3e7c8c5ec2 100644 --- a/source/libs/index/test/fstTest.cc +++ b/source/libs/index/test/fstTest.cc @@ -49,7 +49,6 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size, const std::string& fileName = "/tmp/tindex.tindex") { - tfInit(); _wc = writerCtxCreate(TFile, fileName.c_str(), true, 64 * 1024); _w = fstCountingWriterCreate(_wc); _size = size; @@ -116,7 +115,6 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); } private: @@ -170,7 +168,6 @@ void Performance_fstReadRecords(FstReadMemory* m) { } void checkMillonWriteAndReadOfFst() { - tfInit(); FstWriter* fw = new FstWriter; Performance_fstWriteRecords(fw); delete fw; @@ -181,11 +178,9 @@ void checkMillonWriteAndReadOfFst() { } Performance_fstReadRecords(fr); - tfCleanup(); delete fr; } void checkFstLongTerm() { - tfInit(); FstWriter* fw = new FstWriter; // Performance_fstWriteRecords(fw); @@ -235,12 +230,10 @@ void checkFstLongTerm() { // for (int i = 0; i < result.size(); i++) { // assert(result[i] == i); // check result //} - tfCleanup(); // free(ctx); // delete m; } void checkFstCheckIterator() { - tfInit(); FstWriter* fw = new FstWriter; int64_t s = taosGetTimestampUs(); int count = 2; @@ -275,7 +268,6 @@ void checkFstCheckIterator() { free(ctx); delete m; - tfCleanup(); } void fst_get(Fst* fst) { @@ -294,8 +286,6 @@ void fst_get(Fst* fst) { #define NUM_OF_THREAD 10 void validateTFile(char* arg) { - tfInit(); - std::thread threads[NUM_OF_THREAD]; // std::vector threads; TFileReader* reader = tfileReaderOpen(arg, 0, 20000000, "tag1"); @@ -309,12 +299,9 @@ void validateTFile(char* arg) { // wait join threads[i].join(); } - tfCleanup(); } void iterTFileReader(char* path, char* ver) { - tfInit(); - int version = atoi(ver); TFileReader* reader = tfileReaderOpen(path, 0, version, "tag1"); Iterate* iter = tfileIteratorCreate(reader); @@ -331,7 +318,6 @@ void iterTFileReader(char* path, char* ver) { printf("total size: %d\n term count: %d\n", count, termCount); tfileIteratorDestroy(iter); - tfCleanup(); } int main(int argc, char* argv[]) { diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index d10f5f47b8d5e0b4980efacb78e0fd5271a187b2..2d3d2f8894415236bb6e79849137fdc931b9bc94 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -15,7 +15,7 @@ #include "tglobal.h" #include "tskiplist.h" #include "tutil.h" -#include "ulog.h" +#include "tlog.h" static std::string dir = "/tmp/index"; @@ -24,14 +24,12 @@ static char tindex[PATH_MAX] = {0}; static char tindexDir[PATH_MAX] = {0}; static void EnvInit() { - tfInit(); - std::string path = dir; taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); // init log file - snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str()); - if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("tindex.idx", 1) != 0) { printf("failed to init log"); } // init index file @@ -99,6 +97,7 @@ class FstReadMemory { fstSliceDestroy(&skey); return ok; } + bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) { int64_t s = taosGetTimestampUs(); bool ok = this->Get(key, val); @@ -120,8 +119,6 @@ class FstReadMemory { printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); swsResultDestroy(rt); } - for (size_t i = 0; i < result.size(); i++) { - } std::cout << std::endl; return true; } @@ -137,7 +134,6 @@ class FstReadMemory { fstDestroy(_fst); fstSliceDestroy(&_s); writerCtxDestroy(_wc, false); - tfCleanup(); } private: @@ -196,6 +192,10 @@ class TFst { } return fr->Get(k, v); } + bool Search(AutomationCtx* ctx, std::vector& result) { + // add more + return fr->Search(ctx, result); + } private: FstWriter* fw; @@ -229,5 +229,9 @@ TEST_F(FstEnv, writeNormal) { assert(fst->Get("a", &val) == false); assert(fst->Get("aa", &val) == true); assert(val == 0); + + std::vector rlt; + AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_ALWAYS); + assert(fst->Search(ctx, rlt) == true); } -TEST_F(FstEnv, writeExcpet) {} +TEST_F(FstEnv, WriteMillonrRecord) {} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index bbcc654ae26fe8b4ffbbf01072c893bbea067876..a50e91b0944870d4c658c5d51ec38028aa0d98a6 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -420,7 +420,6 @@ class IndexTFileEnv : public ::testing::Test { virtual void SetUp() { taosRemoveDir(dir.c_str()); taosMkDir(dir.c_str()); - tfInit(); fObj = new TFileObj(dir, colName); } @@ -428,7 +427,6 @@ class IndexTFileEnv : public ::testing::Test { // indexClose(index); // indexeptsDestroy(opts); delete fObj; - tfCleanup(); // tfileWriterDestroy(twrite); } TFileObj* fObj; @@ -800,13 +798,10 @@ class IndexObj { class IndexEnv2 : public ::testing::Test { protected: virtual void SetUp() { - tfInit(); index = new IndexObj(); - // } virtual void TearDown() { delete index; - tfCleanup(); } IndexObj* index; }; diff --git a/source/libs/index/test/utilUT.cc b/source/libs/index/test/utilUT.cc new file mode 100644 index 0000000000000000000000000000000000000000..aeff20d488d9c99805a3cdb250309d7fdb038cf3 --- /dev/null +++ b/source/libs/index/test/utilUT.cc @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include +#include +#include "index.h" +#include "indexInt.h" +#include "index_cache.h" +#include "index_fst.h" +#include "index_fst_counting_writer.h" +#include "index_fst_util.h" +#include "index_tfile.h" +#include "index_util.h" +#include "tglobal.h" +#include "tskiplist.h" +#include "tutil.h" + +class UtilEnv : public ::testing::Test { + protected: + virtual void SetUp() { + src = (SArray *)taosArrayInit(2, sizeof(void *)); + for (int i = 0; i < 3; i++) { + SArray *m = taosArrayInit(10, sizeof(uint64_t)); + taosArrayPush(src, &m); + } + + rslt = (SArray *)taosArrayInit(10, sizeof(uint64_t)); + } + virtual void TearDown() { + for (int i = 0; i < taosArrayGetSize(src); i++) { + SArray *m = (SArray *)taosArrayGetP(src, i); + taosArrayDestroy(m); + } + taosArrayDestroy(src); + } + + SArray *src; + SArray *rslt; +}; + +static void clearSourceArray(SArray *p) { + for (int i = 0; i < taosArrayGetSize(p); i++) { + SArray *m = (SArray *)taosArrayGetP(p, i); + taosArrayClear(m); + } +} +static void clearFinalArray(SArray *p) { taosArrayClear(p); } +TEST_F(UtilEnv, intersectionSimpleResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 10); + + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectMultiEmptyResult) { + SArray *f = (SArray *)taosArrayGetP(src, 0); + for (int i = 10; i < 20; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < 10; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 20; i < 30; i++) { + uint64_t val = i; + taosArrayPush(f, &val); + } + // empty source + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); + clearSourceArray(src); + clearFinalArray(rslt); +} +TEST_F(UtilEnv, intersectSimpleEmpty) { + clearSourceArray(src); + clearFinalArray(rslt); + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, intersect01) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {2, 3, 4, 5}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {1, 2, 3, 5}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {3, 5, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 2); +} +TEST_F(UtilEnv, intersect02) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {8, 10, 12, 13}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {9, 10, 11}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + + iIntersection(src, rslt); + assert(taosArrayGetSize(rslt) == 0); +} +TEST_F(UtilEnv, 01union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 02union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 14, 15}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {13, 14, 15}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 3); +} +TEST_F(UtilEnv, 03union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {13, 16, 18, 20}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {0, 12, 13, 20, 23}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + uint64_t arr3[] = {1, 12, 13, 16, 17}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 9); +} +TEST_F(UtilEnv, 04union) { + clearSourceArray(src); + clearFinalArray(rslt); + + uint64_t arr1[] = {1, 4, 5, 6}; + SArray * f = (SArray *)taosArrayGetP(src, 0); + for (int i = 0; i < sizeof(arr1) / sizeof(arr1[0]); i++) { + taosArrayPush(f, &arr1[i]); + } + + uint64_t arr2[] = {7, 8, 10}; + f = (SArray *)taosArrayGetP(src, 1); + for (int i = 0; i < sizeof(arr2) / sizeof(arr2[0]); i++) { + taosArrayPush(f, &arr2[i]); + } + + uint64_t arr3[] = {20, 21, 30, 100, 120}; + f = (SArray *)taosArrayGetP(src, 2); + for (int i = 0; i < sizeof(arr3) / sizeof(arr3[0]); i++) { + taosArrayPush(f, &arr3[i]); + } + iUnion(src, rslt); + assert(taosArrayGetSize(rslt) == 12); +} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 18a316320e839e2e0a74e5e7dba15a449677c82a..e23c9ebe9d21fb8842b313f3ad886d2f813dba5d 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -72,6 +72,8 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_CHAR_ARRAY_FIELD(tableAlias); COPY_CHAR_ARRAY_FIELD(colName); // COPY_NODE_FIELD(pProjectRef); + COPY_SCALAR_FIELD(dataBlockId); + COPY_SCALAR_FIELD(slotId); return (SNode*)pDst; } @@ -142,16 +144,8 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { return (SNode*)pDst; } -static SNode* columnRefNodeCopy(const SColumnRefNode* pSrc, SColumnRefNode* pDst) { - dataTypeCopy(&pSrc->dataType, &pDst->dataType); - COPY_SCALAR_FIELD(tupleId); - COPY_SCALAR_FIELD(slotId); - COPY_SCALAR_FIELD(columnId); - return (SNode*)pDst; -} - static SNode* targetNodeCopy(const STargetNode* pSrc, STargetNode* pDst) { - COPY_SCALAR_FIELD(tupleId); + COPY_SCALAR_FIELD(dataBlockId); COPY_SCALAR_FIELD(slotId); COPY_NODE_FIELD(pExpr); return (SNode*)pDst; @@ -183,8 +177,6 @@ SNode* nodesCloneNode(const SNode* pNode) { return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); case QUERY_NODE_FUNCTION: return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); - case QUERY_NODE_COLUMN_REF: - return columnRefNodeCopy((const SColumnRefNode*)pNode, (SColumnRefNode*)pDst); case QUERY_NODE_TARGET: return targetNodeCopy((const STargetNode*)pNode, (STargetNode*)pDst); case QUERY_NODE_REAL_TABLE: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 356deb2f513cad1962bbecb8158e8bec8f838257..b1fd4bb56f78c63384a79d9955550f9a0690cb96 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -20,6 +20,9 @@ #include "tjson.h" static int32_t nodeToJson(const void* pObj, SJson* pJson); +static int32_t jsonToNode(const SJson* pJson, void* pObj); +static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** pNode); +static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode); static char* nodeName(ENodeType type) { switch (type) { @@ -55,13 +58,11 @@ static char* nodeName(ENodeType type) { return "NodeList"; case QUERY_NODE_FILL: return "Fill"; - case QUERY_NODE_COLUMN_REF: - return "ColumnRef"; case QUERY_NODE_TARGET: return "Target"; case QUERY_NODE_RAW_EXPR: return "RawExpr"; - case QUERY_NODE_TUPLE_DESC: + case QUERY_NODE_DATABLOCK_DESC: return "TupleDesc"; case QUERY_NODE_SLOT_DESC: return "SlotDesc"; @@ -85,6 +86,10 @@ static char* nodeName(ENodeType type) { return "PhysiTableScan"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return "PhysiJoin"; + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return "PhysiAgg"; default: break; } @@ -93,7 +98,7 @@ static char* nodeName(ENodeType type) { return tmp; } -static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const SNodeList* pList) { +static int32_t nodeListToJson(SJson* pJson, const char* pName, const SNodeList* pList) { if (LIST_LENGTH(pList) > 0) { SJson* jList = tjsonAddArrayToObject(pJson, pName); if (NULL == jList) { @@ -101,7 +106,7 @@ static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const } SNode* pNode; FOREACH(pNode, pList) { - int32_t code = tjsonAddItem(jList, func, pNode); + int32_t code = tjsonAddItem(jList, nodeToJson, pNode); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -110,6 +115,31 @@ static int32_t addNodeList(SJson* pJson, const char* pName, FToJson func, const return TSDB_CODE_SUCCESS; } +static int32_t jsonToNodeList(const SJson* pJson, const char* pName, SNodeList** pList) { + const SJson* pJsonArray = tjsonGetObjectItem(pJson, pName); + int32_t size = (NULL == pJsonArray ? 0 : tjsonGetArraySize(pJsonArray)); + if (size > 0) { + *pList = nodesMakeList(); + if (NULL == *pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + int32_t code = TSDB_CODE_SUCCESS; + for (int32_t i = 0; i < size; ++i) { + SJson* pJsonItem = tjsonGetArrayItem(pJsonArray, i); + SNode* pNode = NULL; + code = makeNodeByJson(pJsonItem, &pNode); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListAppend(*pList, pNode); + } + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + static const char* jkTableMetaUid = "TableMetaUid"; static const char* jkTableMetaSuid = "TableMetaSuid"; @@ -134,13 +164,13 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkLogicPlanId, pNode->id); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkLogicPlanTargets, nodeToJson, pNode->pTargets); + code = nodeListToJson(pJson, jkLogicPlanTargets, pNode->pTargets); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkLogicPlanConditions, nodeToJson, pNode->pConditions); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkLogicPlanChildren, nodeToJson, pNode->pChildren); + code = nodeListToJson(pJson, jkLogicPlanChildren, pNode->pChildren); } return code; @@ -154,7 +184,7 @@ static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) { int32_t code = logicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkScanLogicPlanScanCols, nodeToJson, pNode->pScanCols); + code = nodeListToJson(pJson, jkScanLogicPlanScanCols, pNode->pScanCols); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanLogicPlanTableMeta, tableMetaToJson, pNode->pMeta); @@ -170,7 +200,7 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) { int32_t code = logicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkProjectLogicPlanProjections, nodeToJson, pNode->pProjections); + code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections); } return code; @@ -193,19 +223,33 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) { return code; } -static const char* jkPhysiPlanOutputTuple = "OutputTuple"; +static const char* jkPhysiPlanOutputDataBlockDesc = "OutputDataBlockDesc"; static const char* jkPhysiPlanConditions = "Conditions"; static const char* jkPhysiPlanChildren = "Children"; static int32_t physicPlanNodeToJson(const void* pObj, SJson* pJson) { const SPhysiNode* pNode = (const SPhysiNode*)pObj; - int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputTuple, nodeToJson, &pNode->outputTuple); + int32_t code = tjsonAddObject(pJson, jkPhysiPlanOutputDataBlockDesc, nodeToJson, &pNode->outputDataBlockDesc); if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkPhysiPlanConditions, nodeToJson, pNode->pConditions); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkPhysiPlanChildren, nodeToJson, pNode->pChildren); + code = nodeListToJson(pJson, jkPhysiPlanChildren, pNode->pChildren); + } + + return code; +} + +static int32_t jsonToPhysicPlanNode(const SJson* pJson, void* pObj) { + SPhysiNode* pNode = (SPhysiNode*)pObj; + + int32_t code = tjsonToObject(pJson, jkPhysiPlanOutputDataBlockDesc, jsonToNode, &pNode->outputDataBlockDesc); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkPhysiPlanConditions, &pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkPhysiPlanChildren, &pNode->pChildren); } return code; @@ -223,7 +267,7 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkScanPhysiPlanScanCols, nodeToJson, pNode->pScanCols); + code = nodeListToJson(pJson, jkScanPhysiPlanScanCols, pNode->pScanCols); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableId, pNode->uid); @@ -244,10 +288,40 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { + STagScanPhysiNode* pNode = (STagScanPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkScanPhysiPlanScanCols, &pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkScanPhysiPlanTableId, &pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanOrder, &pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanCount, &pNode->count); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkScanPhysiPlanReverseScanCount, &pNode->reverse); + } + + return code; +} + static int32_t physiTagScanNodeToJson(const void* pObj, SJson* pJson) { return physiScanNodeToJson(pObj, pJson); } +static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { + return jsonToPhysiScanNode(pJson, pObj); +} + static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; static const char* jkTableScanPhysiPlanStartKey = "StartKey"; static const char* jkTableScanPhysiPlanEndKey = "EndKey"; @@ -269,6 +343,23 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { + STableScanPhysiNode* pNode = (STableScanPhysiNode*)pObj; + + int32_t code = jsonToPhysiScanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanFlag, &pNode->scanFlag); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanStartKey, &pNode->scanRange.skey); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanEndKey, &pNode->scanRange.ekey); + } + + return code; +} + static const char* jkProjectPhysiPlanProjections = "Projections"; static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { @@ -276,7 +367,96 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) { int32_t code = physicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkProjectPhysiPlanProjections, nodeToJson, pNode->pProjections); + code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections); + } + + return code; +} + +static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) { + SProjectPhysiNode* pNode = (SProjectPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections); + } + + return code; +} + +static const char* jkJoinPhysiPlanJoinType = "JoinType"; +static const char* jkJoinPhysiPlanOnConditions = "OnConditions"; +static const char* jkJoinPhysiPlanTargets = "Targets"; + +static int32_t physiJoinNodeToJson(const void* pObj, SJson* pJson) { + const SJoinPhysiNode* pNode = (const SJoinPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOnConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets); + } + + return code; +} + +static int32_t jsonToPhysiJoinNode(const SJson* pJson, void* pObj) { + SJoinPhysiNode* pNode = (SJoinPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + int32_t val; + code = tjsonGetIntValue(pJson, jkJoinPhysiPlanJoinType, &val); + pNode->joinType = val; + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOnConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets); + } + + return code; +} + +static const char* jkAggPhysiPlanExprs = "Exprs"; +static const char* jkAggPhysiPlanGroupKeys = "GroupKeys"; +static const char* jkAggPhysiPlanAggFuncs = "AggFuncs"; + +static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) { + const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkAggPhysiPlanExprs, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkAggPhysiPlanGroupKeys, pNode->pGroupKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkAggPhysiPlanAggFuncs, pNode->pAggFuncs); + } + + return code; +} + +static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) { + SAggPhysiNode* pNode = (SAggPhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkAggPhysiPlanExprs, &pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkAggPhysiPlanGroupKeys, &pNode->pGroupKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkAggPhysiPlanAggFuncs, &pNode->pAggFuncs); } return code; @@ -290,10 +470,10 @@ static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) { int32_t code = logicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkAggLogicPlanGroupKeys, nodeToJson, pNode->pGroupKeys); + code = nodeListToJson(pJson, jkAggLogicPlanGroupKeys, pNode->pGroupKeys); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkAggLogicPlanAggFuncs, nodeToJson, pNode->pAggFuncs); + code = nodeListToJson(pJson, jkAggLogicPlanAggFuncs, pNode->pAggFuncs); } return code; @@ -321,6 +501,23 @@ static int32_t dataTypeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToDataType(const SJson* pJson, void* pObj) { + SDataType* pNode = (SDataType*)pObj; + + int32_t code = tjsonGetUTinyIntValue(pJson, jkDataTypeType, &pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkDataTypePrecision, &pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkDataTypeScale, &pNode->scale); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkDataTypeDataBytes, &pNode->bytes); + } + + return TSDB_CODE_SUCCESS; +} + static const char* jkExprDataType = "DataType"; static const char* jkExprAliasName = "AliasName"; @@ -335,6 +532,17 @@ static int32_t exprNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToExprNode(const SJson* pJson, void* pObj) { + SExprNode* pNode = (SExprNode*)pObj; + + int32_t code = tjsonToObject(pJson, jkExprDataType, jsonToDataType, &pNode->resType); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkExprAliasName, pNode->aliasName); + } + + return code; +} + static const char* jkColumnTableId = "TableId"; static const char* jkColumnColId = "ColId"; static const char* jkColumnColType = "ColType"; @@ -342,6 +550,8 @@ static const char* jkColumnDbName = "DbName"; static const char* jkColumnTableName = "TableName"; static const char* jkColumnTableAlias = "TableAlias"; static const char* jkColumnColName = "ColName"; +static const char* jkColumnDataBlockId = "DataBlockId"; +static const char* jkColumnSlotId = "SlotId"; static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { const SColumnNode* pNode = (const SColumnNode*)pObj; @@ -368,6 +578,49 @@ static int32_t columnNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddStringToObject(pJson, jkColumnColName, pNode->colName); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnDataBlockId, pNode->dataBlockId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkColumnSlotId, pNode->slotId); + } + + return code; +} + +static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { + SColumnNode* pNode = (SColumnNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUBigIntValue(pJson, jkColumnTableId, &pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkColumnColId, &pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + int32_t tmp; + code = tjsonGetIntValue(pJson, jkColumnColType, &tmp); + pNode->colType = tmp; + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnTableName, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnTableAlias, pNode->tableAlias); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkColumnColName, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkColumnDataBlockId, &pNode->dataBlockId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetSmallIntValue(pJson, jkColumnSlotId, &pNode->slotId); + } return code; } @@ -384,7 +637,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->isDuration); + code = tjsonAddBoolToObject(pJson, jkValueDuration, pNode->isDuration); } switch (pNode->node.resType.type) { case TSDB_DATA_TYPE_NULL: @@ -426,6 +679,56 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { + SValueNode* pNode = (SValueNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->isDuration); + } + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + code = tjsonGetBigIntValue(pJson, jkValueDuration, &pNode->datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + code = tjsonGetUBigIntValue(pJson, jkValueDuration, &pNode->datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + code = tjsonGetDoubleValue(pJson, jkValueDuration, &pNode->datum.d); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->datum.p); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + static const char* jkOperatorType = "OpType"; static const char* jkOperatorLeft = "Left"; static const char* jkOperatorRight = "Right"; @@ -447,6 +750,25 @@ static int32_t operatorNodeToJson(const void* pObj, SJson* pJson) { return code; } +static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { + SOperatorNode* pNode = (SOperatorNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + int32_t val; + code = tjsonGetIntValue(pJson, jkOperatorType, &val); + pNode->opType = val; + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkOperatorRight, &pNode->pRight); + } + + return code; +} + static const char* jkLogicCondType = "CondType"; static const char* jkLogicCondParameters = "Parameters"; @@ -458,7 +780,23 @@ static int32_t logicConditionNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkLogicCondType, pNode->condType); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkLogicCondParameters, nodeToJson, pNode->pParameterList); + code = nodeListToJson(pJson, jkLogicCondParameters, pNode->pParameterList); + } + + return code; +} + +static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { + SLogicConditionNode* pNode = (SLogicConditionNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + int32_t val; + code = tjsonGetIntValue(pJson, jkLogicCondType, &val); + pNode->condType = val; + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); } return code; @@ -483,7 +821,27 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) { code = tjsonAddIntegerToObject(pJson, jkFunctionType, pNode->funcType); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkFunctionParameter, nodeToJson, pNode->pParameterList); + code = nodeListToJson(pJson, jkFunctionParameter, pNode->pParameterList); + } + + return code; +} + +static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) { + SFunctionNode* pNode = (SFunctionNode*)pObj; + + int32_t code = jsonToExprNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetStringValue(pJson, jkFunctionName, pNode->functionName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionId, &pNode->funcId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkFunctionType, &pNode->funcType); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkFunctionParameter, &pNode->pParameterList); } return code; @@ -497,47 +855,39 @@ static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkGroupingSetType, pNode->groupingSetType); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkGroupingSetParameter, nodeToJson, pNode->pParameterList); + code = nodeListToJson(pJson, jkGroupingSetParameter, pNode->pParameterList); } return code; } -static const char* jkColumnRefDataType = "DataType"; -static const char* jkColumnRefTupleId = "TupleId"; -static const char* jkColumnRefSlotId = "SlotId"; -static const char* jkColumnRefColumnId = "ColumnId"; +static const char* jkTargetDataBlockId = "DataBlockId"; +static const char* jkTargetSlotId = "SlotId"; +static const char* jkTargetExpr = "Expr"; -static int32_t columnRefNodeToJson(const void* pObj, SJson* pJson) { - const SColumnRefNode* pNode = (const SColumnRefNode*)pObj; +static int32_t targetNodeToJson(const void* pObj, SJson* pJson) { + const STargetNode* pNode = (const STargetNode*)pObj; - int32_t code = tjsonAddObject(pJson, jkColumnRefDataType, dataTypeToJson, &pNode->dataType); - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefTupleId, pNode->tupleId); - } + int32_t code = tjsonAddIntegerToObject(pJson, jkTargetDataBlockId, pNode->dataBlockId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefSlotId, pNode->slotId); + code = tjsonAddIntegerToObject(pJson, jkTargetSlotId, pNode->slotId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkColumnRefColumnId, pNode->columnId); + code = tjsonAddObject(pJson, jkTargetExpr, nodeToJson, pNode->pExpr); } return code; } -static const char* jkTargetTupleId = "TupleId"; -static const char* jkTargetSlotId = "SlotId"; -static const char* jkTargetExpr = "Expr"; - -static int32_t targetNodeToJson(const void* pObj, SJson* pJson) { - const STargetNode* pNode = (const STargetNode*)pObj; +static int32_t jsonToTargetNode(const SJson* pJson, void* pObj) { + STargetNode* pNode = (STargetNode*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkTargetTupleId, pNode->tupleId); + int32_t code = tjsonGetSmallIntValue(pJson, jkTargetDataBlockId, &pNode->dataBlockId); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkTargetSlotId, pNode->slotId); + code = tjsonGetSmallIntValue(pJson, jkTargetSlotId, &pNode->slotId); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkTargetExpr, nodeToJson, pNode->pExpr); + code = jsonToNodeObject(pJson, jkTargetExpr, &pNode->pExpr); } return code; @@ -545,6 +895,8 @@ static int32_t targetNodeToJson(const void* pObj, SJson* pJson) { static const char* jkSlotDescSlotId = "SlotId"; static const char* jkSlotDescDataType = "DataType"; +static const char* jkSlotDescReserve = "Reserve"; +static const char* jkSlotDescOutput = "Output"; static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; @@ -553,19 +905,53 @@ static int32_t slotDescNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSlotDescDataType, dataTypeToJson, &pNode->dataType); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSlotDescReserve, pNode->reserve); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSlotDescOutput, pNode->output); + } + + return code; +} + +static int32_t jsonToSlotDescNode(const SJson* pJson, void* pObj) { + SSlotDescNode* pNode = (SSlotDescNode*)pObj; + + int32_t code = tjsonGetSmallIntValue(pJson, jkSlotDescSlotId, &pNode->slotId); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkSlotDescDataType, jsonToDataType, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSlotDescReserve, &pNode->reserve); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSlotDescOutput, &pNode->output); + } return code; } -static const char* jkTupleDescTupleId = "TupleId"; -static const char* jkTupleDescSlots = "Slots"; +static const char* jkDataBlockDescDataBlockId = "DataBlockId"; +static const char* jkDataBlockDescSlots = "Slots"; -static int32_t tupleDescNodeToJson(const void* pObj, SJson* pJson) { - const STupleDescNode* pNode = (const STupleDescNode*)pObj; +static int32_t dataBlockDescNodeToJson(const void* pObj, SJson* pJson) { + const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj; - int32_t code = tjsonAddIntegerToObject(pJson, jkTupleDescTupleId, pNode->tupleId); + int32_t code = tjsonAddIntegerToObject(pJson, jkDataBlockDescDataBlockId, pNode->dataBlockId); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkTupleDescSlots, nodeToJson, pNode->pSlots); + code = nodeListToJson(pJson, jkDataBlockDescSlots, pNode->pSlots); + } + + return code; +} + +static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) { + SDataBlockDescNode* pNode = (SDataBlockDescNode*)pObj; + + int32_t code = tjsonGetSmallIntValue(pJson, jkDataBlockDescDataBlockId, &pNode->dataBlockId); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkDataBlockDescSlots, &pNode->pSlots); } return code; @@ -588,7 +974,7 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct); if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkSelectStmtProjections, nodeToJson, pNode->pProjectionList); + code = nodeListToJson(pJson, jkSelectStmtProjections, pNode->pProjectionList); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtFrom, nodeToJson, pNode->pFromTable); @@ -597,19 +983,19 @@ static int32_t selectStmtTojson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkSelectStmtWhere, nodeToJson, pNode->pWhere); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkSelectStmtPartitionBy, nodeToJson, pNode->pPartitionByList); + code = nodeListToJson(pJson, jkSelectStmtPartitionBy, pNode->pPartitionByList); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkSelectStmtGroupBy, nodeToJson, pNode->pGroupByList); + code = nodeListToJson(pJson, jkSelectStmtGroupBy, pNode->pGroupByList); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtHaving, nodeToJson, pNode->pHaving); } if (TSDB_CODE_SUCCESS == code) { - code = addNodeList(pJson, jkSelectStmtOrderBy, nodeToJson, pNode->pOrderByList); + code = nodeListToJson(pJson, jkSelectStmtOrderBy, pNode->pOrderByList); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSelectStmtLimit, nodeToJson, pNode->pLimit); @@ -646,14 +1032,12 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_NODE_LIST: case QUERY_NODE_FILL: - case QUERY_NODE_COLUMN_REF: - return columnRefNodeToJson(pObj, pJson); case QUERY_NODE_TARGET: return targetNodeToJson(pObj, pJson); case QUERY_NODE_RAW_EXPR: break; - case QUERY_NODE_TUPLE_DESC: - return tupleDescNodeToJson(pObj, pJson); + case QUERY_NODE_DATABLOCK_DESC: + return dataBlockDescNodeToJson(pObj, pJson); case QUERY_NODE_SLOT_DESC: return slotDescNodeToJson(pObj, pJson); case QUERY_NODE_SET_OPERATOR: @@ -676,6 +1060,73 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiTableScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return physiProjectNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return physiJoinNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return physiAggNodeToJson(pObj, pJson); + default: + break; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { + switch (nodeType(pObj)) { + case QUERY_NODE_COLUMN: + return jsonToColumnNode(pJson, pObj); + case QUERY_NODE_VALUE: + return jsonToValueNode(pJson, pObj); + case QUERY_NODE_OPERATOR: + return jsonToOperatorNode(pJson, pObj); + case QUERY_NODE_LOGIC_CONDITION: + return jsonToLogicConditionNode(pJson, pObj); + case QUERY_NODE_FUNCTION: + return jsonToFunctionNode(pJson, pObj); + // case QUERY_NODE_REAL_TABLE: + // case QUERY_NODE_TEMP_TABLE: + // case QUERY_NODE_JOIN_TABLE: + // break; + // case QUERY_NODE_GROUPING_SET: + // return jsonToGroupingSetNode(pJson, pObj); + // case QUERY_NODE_ORDER_BY_EXPR: + // case QUERY_NODE_LIMIT: + // case QUERY_NODE_STATE_WINDOW: + // case QUERY_NODE_SESSION_WINDOW: + // case QUERY_NODE_INTERVAL_WINDOW: + // case QUERY_NODE_NODE_LIST: + // case QUERY_NODE_FILL: + case QUERY_NODE_TARGET: + return jsonToTargetNode(pJson, pObj); + // case QUERY_NODE_RAW_EXPR: + // break; + case QUERY_NODE_DATABLOCK_DESC: + return jsonToDataBlockDescNode(pJson, pObj); + case QUERY_NODE_SLOT_DESC: + return jsonToSlotDescNode(pJson, pObj); + // case QUERY_NODE_SET_OPERATOR: + // break; + // case QUERY_NODE_SELECT_STMT: + // return jsonToSelectStmt(pJson, pObj); + // case QUERY_NODE_SHOW_STMT: + // break; + // case QUERY_NODE_LOGIC_PLAN_SCAN: + // return jsonToLogicScanNode(pJson, pObj); + // case QUERY_NODE_LOGIC_PLAN_JOIN: + // return jsonToLogicJoinNode(pJson, pObj); + // case QUERY_NODE_LOGIC_PLAN_AGG: + // return jsonToLogicAggNode(pJson, pObj); + // case QUERY_NODE_LOGIC_PLAN_PROJECT: + // return jsonToLogicProjectNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return jsonToPhysiTagScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + return jsonToPhysiTableScanNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return jsonToPhysiProjectNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return jsonToPhysiJoinNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return jsonToPhysiAggNode(pJson, pObj); default: break; } @@ -683,18 +1134,57 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { } static const char* jkNodeType = "Type"; +static const char* jkNodeName = "Name"; + static int32_t nodeToJson(const void* pObj, SJson* pJson) { const SNode* pNode = (const SNode*)pObj; - char* pNodeName = nodeName(nodeType(pNode)); - int32_t code = tjsonAddStringToObject(pJson, jkNodeType, pNodeName); + int32_t code = tjsonAddIntegerToObject(pJson, jkNodeType, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkNodeName, nodeName(pNode->type)); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, nodeName(pNode->type), specificNodeToJson, pNode); + } + + return code; +} + +static int32_t jsonToNode(const SJson* pJson, void* pObj) { + SNode* pNode = (SNode*)pObj; + + int32_t val = 0; + int32_t code = tjsonGetIntValue(pJson, jkNodeType, &val); + pNode->type = val; + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, nodeName(pNode->type), jsonToSpecificNode, pNode); + } + + return code; +} + +static int32_t makeNodeByJson(const SJson* pJson, SNode** pNode) { + int32_t val = 0; + int32_t code = tjsonGetIntValue(pJson, jkNodeType, &val); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, pNodeName, specificNodeToJson, pNode); + *pNode = nodesMakeNode(val); + if (NULL == *pNode) { + return TSDB_CODE_FAILED; + } + code = jsonToNode(pJson, *pNode); } return code; } +static int32_t jsonToNodeObject(const SJson* pJson, const char* pName, SNode** pNode) { + SJson* pJsonNode = tjsonGetObjectItem(pJson, pName); + if (NULL == pJsonNode) { + return TSDB_CODE_FAILED; + } + return makeNodeByJson(pJsonNode, pNode); +} + int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen) { if (NULL == pNode || NULL == pStr || NULL == pLen) { return TSDB_CODE_SUCCESS; @@ -720,5 +1210,18 @@ int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* } int32_t nodesStringToNode(const char* pStr, SNode** pNode) { + if (NULL == pStr || NULL == pNode) { + return TSDB_CODE_SUCCESS; + } + SJson* pJson = tjsonParse(pStr); + if (NULL == pJson) { + return TSDB_CODE_FAILED; + } + int32_t code = makeNodeByJson(pJson, pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(*pNode); + terrno = code; + return code; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index e61375b2023670db09338a8bd7a844e22c5f5788..5c81cce94aa66f8b082e686c509fef483eff13e2 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -105,6 +105,9 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_RAW_EXPR: res = walkNode(((SRawExprNode*)pNode)->pNode, order, walker, pContext); break; + case QUERY_NODE_TARGET: + res = walkNode(((STargetNode*)pNode)->pExpr, order, walker, pContext); + break; default: break; } @@ -228,6 +231,9 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_RAW_EXPR: res = rewriteNode(&(((SRawExprNode*)pNode)->pNode), order, rewriter, pContext); break; + case QUERY_NODE_TARGET: + res = rewriteNode(&(((STargetNode*)pNode)->pExpr), order, rewriter, pContext); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 5117fe6f54e13ee4365a21d1b7c1e20eecfcf614..eac5606bd62e2a3d4b1868359483d6e30caca90c 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -63,8 +63,6 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SNodeListNode)); case QUERY_NODE_FILL: return makeNode(type, sizeof(SFillNode)); - case QUERY_NODE_COLUMN_REF: - return makeNode(type, sizeof(SColumnRefNode)); case QUERY_NODE_RAW_EXPR: return makeNode(type, sizeof(SRawExprNode)); case QUERY_NODE_SET_OPERATOR: @@ -83,8 +81,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SProjectLogicNode)); case QUERY_NODE_TARGET: return makeNode(type, sizeof(STargetNode)); - case QUERY_NODE_TUPLE_DESC: - return makeNode(type, sizeof(STupleDescNode)); + case QUERY_NODE_DATABLOCK_DESC: + return makeNode(type, sizeof(SDataBlockDescNode)); case QUERY_NODE_SLOT_DESC: return makeNode(type, sizeof(SSlotDescNode)); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: @@ -93,26 +91,57 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(STableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_JOIN: + return makeNode(type, sizeof(SJoinPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_AGG: + return makeNode(type, sizeof(SAggPhysiNode)); default: break; } return NULL; } -static EDealRes destroyNode(SNode* pNode, void* pContext) { - switch (nodeType(pNode)) { - case QUERY_NODE_VALUE: - tfree(((SValueNode*)pNode)->literal); +static EDealRes destroyNode(SNode** pNode, void* pContext) { + if (NULL == pNode || NULL == *pNode) { + return DEAL_RES_IGNORE_CHILD; + } + + switch (nodeType(*pNode)) { + case QUERY_NODE_VALUE: { + SValueNode* pValue = (SValueNode*)*pNode; + + tfree(pValue->literal); + if (IS_VAR_DATA_TYPE(pValue->node.resType.type)) { + tfree(pValue->datum.p); + } + + break; + } + case QUERY_NODE_LOGIC_CONDITION: + nodesDestroyList(((SLogicConditionNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_FUNCTION: + nodesDestroyList(((SFunctionNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_GROUPING_SET: + nodesDestroyList(((SGroupingSetNode*)(*pNode))->pParameterList); + break; + case QUERY_NODE_NODE_LIST: + nodesDestroyList(((SNodeListNode*)(*pNode))->pNodeList); break; default: break; } - tfree(pNode); + tfree(*pNode); return DEAL_RES_CONTINUE; } void nodesDestroyNode(SNode* pNode) { - nodesWalkNodePostOrder(pNode, destroyNode, NULL); + if (NULL == pNode) { + return; + } + + nodesRewriteNodePostOrder(&pNode, destroyNode, NULL); } SNodeList* nodesMakeList() { @@ -189,9 +218,9 @@ SNode* nodesListGetNode(SNodeList* pList, int32_t index) { } void nodesDestroyList(SNodeList* pList) { - SNode* node; - FOREACH(node, pList) { - nodesDestroyNode(node); + SListCell* pNext = pList->pHead; + while (NULL != pNext) { + pNext = nodesListErase(pList, pNext); } tfree(pList); } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 242f659288880de268d3f4edb8b10375ff50527e..fa713d5fb42cd8ddda7ccb3b9037f4ca31f88891 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -449,10 +449,10 @@ tagitemlist1(A) ::= tagitemlist1(X) COMMA tagitem1(Y). { taosArrayPush(X, &Y); A tagitemlist1(A) ::= tagitem1(Y). { A = taosArrayInit(4, sizeof(SToken)); taosArrayPush(A, &Y); } %type tagitem1 {SToken} -tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; } -tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; } +tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} +tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; A.z = X.z;} tagitem1(A) ::= INTEGER(X). { A = X; } tagitem1(A) ::= FLOAT(X). { A = X; } tagitem1(A) ::= STRING(X). { A = X; } diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index b4125b0a24e92c1f3b3266fc980c5e92297cca03..6131d50c127be5c1270cc9e5da18d55c9033137a 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -260,8 +260,11 @@ static int32_t doParseSerializeTagValue(SSchema* pTagSchema, int32_t numOfInputT SKvParam param = {.builder = pKvRowBuilder, .schema = pSchema}; SToken* pItem = taosArrayGet(pTagValList, i); - code = parseValueToken(&endPtr, pItem, pSchema, tsPrecision, tmpTokenBuf, KvRowAppend, ¶m, pMsgBuf); + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && pItem->z[0] == '\'') { + pItem->z += 1; + } + code = parseValueToken(&endPtr, pItem, pSchema, tsPrecision, tmpTokenBuf, KvRowAppend, ¶m, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return buildInvalidOperationMsg(pMsgBuf, msg1); } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 9a2ca2da98399455653c24074368b9df6471e20d..05aebb2cb2c35678512b75079d05656ce40f4399 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -230,7 +230,7 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo) { } void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { - assert(dst != NULL && src != NULL); + assert(dst != NULL && src != NULL/* && src->base.numOfCols > 0*/); *dst = *src; #if 0 @@ -241,8 +241,12 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { #endif dst->pExpr = exprdup(src->pExpr); - dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); - memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); + if (src->base.numOfCols > 0) { + dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); + memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); + } else { + dst->base.pColumns = NULL; + } memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); for (int32_t j = 0; j < src->base.numOfParams; ++j) { diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index ed96737837165ac8c306830d768dc012c5b04431..3ca9a26f1e248709dabdd878dffba409227eae58 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -23,7 +23,6 @@ ** input grammar file: */ #include -#include /************ Begin %include sections from the grammar ************************/ #include @@ -77,10 +76,8 @@ ** zero the stack is dynamically sized using realloc() ** ParseARG_SDECL A static variable declaration for the %extra_argument ** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter ** ParseARG_STORE Code to store %extra_argument into yypParser ** ParseARG_FETCH Code to extract %extra_argument from yypParser -** ParseCTX_* As ParseARG_ except for %extra_context ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. @@ -100,44 +97,38 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 274 +#define YYNOCODE 276 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SVariant yy1; - SField yy16; - int yy40; - SIntervalVal yy52; - int64_t yy61; - SSubclause* yy93; - SWindowStateVal yy112; - SRelationInfo* yy160; - SCreatedTableInfo yy184; - SSqlNode* yy185; - SArray* yy225; - tSqlExpr* yy226; - SCreateDbInfo yy326; - int32_t yy460; - SSessionWindowVal yy463; - SCreateTableSql* yy482; - SLimit yy495; - SCreateAcctInfo yy523; + SSessionWindowVal yy39; + SCreateDbInfo yy42; + SVariant yy43; + int yy44; + tSqlExpr* yy46; + SLimit yy55; + SCreatedTableInfo yy96; + SArray* yy131; + SSqlNode* yy256; + SCreateTableSql* yy272; + SField yy290; + SSubclause* yy303; + int32_t yy310; + SCreateAcctInfo yy341; + int64_t yy459; + SIntervalVal yy530; + SWindowStateVal yy538; + SRelationInfo* yy544; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_PARAM ,pInfo -#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; -#define ParseARG_STORE yypParser->pInfo=pInfo; -#define ParseCTX_SDECL -#define ParseCTX_PDECL -#define ParseCTX_PARAM -#define ParseCTX_FETCH -#define ParseCTX_STORE +#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo +#define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 #define YYNSTATE 368 #define YYNRULE 304 @@ -151,7 +142,6 @@ typedef union { #define YY_MIN_REDUCE 897 #define YY_MAX_REDUCE 1200 /************* End control #defines *******************************************/ -#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -225,246 +215,246 @@ static const YYACTIONTYPE yy_action[] = { /* 40 */ 59, 60, 1176, 1050, 252, 49, 48, 47, 104, 58, /* 50 */ 325, 63, 61, 64, 62, 1022, 21, 1020, 1021, 54, /* 60 */ 53, 1075, 1023, 52, 51, 50, 1024, 206, 1025, 1026, - /* 70 */ 280, 279, 1082, 55, 56, 1044, 59, 60, 1177, 274, + /* 70 */ 280, 279, 1082, 55, 56, 203, 59, 60, 1177, 274, /* 80 */ 252, 49, 48, 47, 89, 58, 325, 63, 61, 64, /* 90 */ 62, 39, 236, 1062, 206, 54, 53, 362, 982, 52, /* 100 */ 51, 50, 27, 55, 57, 1177, 59, 60, 323, 830, /* 110 */ 252, 49, 48, 47, 1075, 58, 325, 63, 61, 64, - /* 120 */ 62, 243, 294, 80, 81, 54, 53, 795, 796, 52, + /* 120 */ 62, 121, 294, 80, 81, 54, 53, 795, 796, 52, /* 130 */ 51, 50, 234, 116, 56, 232, 59, 60, 311, 1047, /* 140 */ 252, 49, 48, 47, 104, 58, 325, 63, 61, 64, - /* 150 */ 62, 42, 776, 361, 360, 54, 53, 952, 359, 52, + /* 150 */ 62, 42, 952, 361, 360, 54, 53, 1038, 359, 52, /* 160 */ 51, 50, 358, 43, 357, 356, 1033, 1034, 30, 1037, /* 170 */ 253, 42, 319, 361, 360, 318, 317, 316, 359, 315, /* 180 */ 314, 313, 358, 312, 357, 356, 310, 1014, 1002, 1003, /* 190 */ 1004, 1005, 1006, 1007, 1008, 1009, 1010, 1011, 1012, 1013, - /* 200 */ 1015, 1016, 1017, 1018, 641, 59, 60, 159, 773, 252, - /* 210 */ 49, 48, 47, 113, 58, 325, 63, 61, 64, 62, - /* 220 */ 1124, 355, 292, 355, 54, 53, 836, 839, 52, 51, - /* 230 */ 50, 282, 206, 54, 53, 7, 321, 52, 51, 50, - /* 240 */ 780, 723, 22, 1177, 591, 592, 593, 594, 595, 596, + /* 200 */ 1015, 1016, 1017, 1018, 641, 59, 60, 159, 113, 252, + /* 210 */ 49, 48, 47, 355, 58, 325, 63, 61, 64, 62, + /* 220 */ 1124, 355, 292, 39, 54, 53, 282, 80, 52, 51, + /* 230 */ 50, 321, 206, 54, 53, 945, 88, 52, 51, 50, + /* 240 */ 776, 158, 22, 1177, 591, 592, 593, 594, 595, 596, /* 250 */ 597, 598, 599, 600, 601, 602, 603, 604, 199, 215, - /* 260 */ 231, 251, 845, 834, 837, 840, 216, 345, 344, 198, - /* 270 */ 195, 193, 175, 174, 172, 217, 80, 321, 83, 330, + /* 260 */ 231, 251, 845, 834, 837, 840, 216, 43, 261, 79, + /* 270 */ 1035, 1046, 175, 174, 172, 217, 275, 124, 723, 330, /* 280 */ 80, 251, 845, 834, 837, 840, 52, 51, 50, 228, - /* 290 */ 229, 121, 78, 326, 63, 61, 64, 62, 759, 756, - /* 300 */ 757, 758, 54, 53, 835, 838, 52, 51, 50, 228, - /* 310 */ 229, 255, 751, 748, 749, 750, 43, 1061, 79, 203, - /* 320 */ 43, 3, 32, 131, 39, 257, 258, 1038, 104, 129, - /* 330 */ 85, 123, 133, 104, 39, 945, 39, 39, 65, 244, - /* 340 */ 245, 158, 273, 39, 86, 39, 843, 746, 747, 305, + /* 290 */ 229, 321, 40, 326, 63, 61, 64, 62, 759, 756, + /* 300 */ 757, 758, 54, 53, 345, 344, 52, 51, 50, 228, + /* 310 */ 229, 255, 751, 748, 749, 750, 746, 747, 836, 839, + /* 320 */ 43, 3, 32, 131, 204, 257, 258, 39, 780, 129, + /* 330 */ 85, 123, 133, 1044, 39, 209, 39, 39, 65, 244, + /* 340 */ 245, 104, 273, 39, 86, 39, 835, 838, 104, 305, /* 350 */ 260, 224, 189, 186, 183, 149, 142, 162, 65, 181, - /* 360 */ 179, 178, 177, 176, 167, 170, 160, 39, 240, 269, - /* 370 */ 39, 88, 1047, 164, 204, 39, 846, 841, 241, 209, + /* 360 */ 179, 178, 177, 176, 167, 170, 160, 39, 269, 77, + /* 370 */ 39, 240, 39, 164, 773, 1047, 846, 841, 241, 243, /* 380 */ 334, 335, 1047, 842, 1047, 1047, 812, 336, 39, 337, - /* 390 */ 256, 1047, 254, 1047, 333, 332, 846, 841, 327, 266, - /* 400 */ 12, 261, 39, 842, 1123, 1035, 84, 261, 270, 82, - /* 410 */ 124, 341, 760, 761, 342, 1047, 127, 92, 1047, 343, - /* 420 */ 365, 364, 190, 1047, 844, 93, 752, 753, 71, 262, - /* 430 */ 35, 259, 347, 340, 339, 275, 1047, 119, 955, 261, - /* 440 */ 792, 946, 802, 803, 158, 74, 811, 158, 1048, 832, - /* 450 */ 1046, 40, 733, 297, 744, 745, 97, 70, 66, 24, - /* 460 */ 735, 299, 734, 868, 847, 70, 300, 250, 40, 40, - /* 470 */ 1049, 72, 640, 14, 77, 13, 67, 117, 67, 23, - /* 480 */ 23, 833, 210, 140, 211, 139, 75, 1171, 23, 4, - /* 490 */ 1170, 16, 18, 15, 17, 764, 765, 762, 763, 147, - /* 500 */ 1134, 146, 20, 1169, 19, 849, 169, 168, 226, 722, - /* 510 */ 227, 207, 208, 212, 205, 1196, 213, 214, 219, 220, - /* 520 */ 1188, 221, 1077, 218, 202, 1133, 238, 44, 1130, 1076, - /* 530 */ 277, 1129, 239, 346, 114, 1116, 1115, 324, 196, 271, - /* 540 */ 791, 76, 281, 1045, 235, 276, 73, 87, 1084, 283, - /* 550 */ 285, 1095, 295, 46, 293, 291, 90, 108, 94, 1092, - /* 560 */ 1093, 1097, 95, 1073, 101, 286, 288, 1117, 105, 106, - /* 570 */ 107, 109, 289, 110, 111, 287, 284, 112, 45, 29, - /* 580 */ 306, 115, 225, 958, 1043, 150, 118, 247, 980, 120, - /* 590 */ 301, 959, 302, 303, 304, 348, 307, 308, 200, 349, - /* 600 */ 151, 38, 322, 350, 954, 957, 130, 953, 331, 1195, - /* 610 */ 137, 351, 1194, 352, 1191, 353, 354, 363, 141, 222, - /* 620 */ 338, 223, 1187, 144, 1186, 1183, 893, 148, 979, 265, - /* 630 */ 264, 892, 268, 891, 874, 873, 41, 31, 8, 70, - /* 640 */ 201, 28, 296, 153, 157, 272, 152, 154, 155, 943, - /* 650 */ 156, 163, 941, 165, 166, 939, 938, 263, 1, 981, - /* 660 */ 937, 171, 936, 173, 935, 934, 933, 932, 931, 930, - /* 670 */ 767, 929, 267, 180, 278, 182, 928, 184, 185, 187, - /* 680 */ 927, 926, 924, 188, 922, 919, 793, 96, 98, 920, - /* 690 */ 194, 917, 804, 197, 918, 913, 99, 100, 2, 798, - /* 700 */ 102, 237, 9, 800, 33, 103, 34, 10, 298, 290, - /* 710 */ 11, 25, 26, 119, 122, 126, 654, 693, 309, 692, - /* 720 */ 36, 125, 689, 37, 687, 128, 686, 685, 683, 682, - /* 730 */ 681, 678, 644, 132, 134, 135, 5, 850, 320, 848, - /* 740 */ 6, 329, 328, 68, 69, 136, 138, 143, 725, 40, - /* 750 */ 145, 724, 721, 670, 668, 660, 666, 662, 664, 658, - /* 760 */ 656, 691, 690, 688, 684, 680, 679, 161, 642, 897, - /* 770 */ 896, 608, 896, 896, 896, 896, 896, 896, 896, 191, + /* 390 */ 256, 1047, 254, 1047, 333, 332, 846, 841, 266, 955, + /* 400 */ 12, 7, 327, 842, 722, 158, 84, 270, 365, 364, + /* 410 */ 190, 341, 760, 761, 342, 1047, 343, 1123, 1047, 946, + /* 420 */ 1047, 198, 195, 193, 82, 158, 752, 753, 83, 262, + /* 430 */ 261, 259, 347, 340, 339, 261, 1047, 119, 843, 127, + /* 440 */ 92, 93, 78, 792, 1048, 802, 811, 803, 71, 74, + /* 450 */ 832, 733, 297, 735, 299, 744, 745, 734, 35, 97, + /* 460 */ 868, 66, 847, 24, 250, 1077, 844, 40, 40, 67, + /* 470 */ 117, 640, 14, 67, 13, 140, 23, 139, 23, 210, + /* 480 */ 70, 70, 833, 16, 211, 15, 4, 23, 169, 168, + /* 490 */ 75, 72, 1171, 1170, 300, 764, 765, 1169, 762, 763, + /* 500 */ 18, 147, 17, 146, 20, 226, 19, 227, 207, 849, + /* 510 */ 1061, 208, 212, 205, 213, 214, 1049, 219, 220, 221, + /* 520 */ 1196, 218, 202, 1188, 1134, 1076, 1133, 238, 44, 1130, + /* 530 */ 277, 1045, 1129, 239, 346, 114, 324, 1116, 196, 1115, + /* 540 */ 271, 276, 791, 87, 1084, 1095, 76, 1092, 90, 281, + /* 550 */ 1093, 235, 283, 1097, 73, 295, 94, 46, 291, 107, + /* 560 */ 95, 1073, 105, 106, 286, 101, 288, 293, 285, 289, + /* 570 */ 110, 108, 109, 1117, 287, 111, 284, 112, 45, 29, + /* 580 */ 115, 225, 1043, 247, 118, 301, 120, 306, 150, 959, + /* 590 */ 302, 303, 304, 980, 307, 958, 308, 200, 38, 322, + /* 600 */ 954, 953, 130, 957, 331, 1195, 348, 137, 1194, 349, + /* 610 */ 1191, 350, 351, 222, 352, 141, 338, 1187, 353, 144, + /* 620 */ 151, 1186, 1183, 354, 148, 223, 979, 363, 893, 265, + /* 630 */ 152, 892, 264, 41, 268, 31, 891, 874, 201, 943, + /* 640 */ 153, 163, 157, 180, 154, 155, 941, 156, 165, 166, + /* 650 */ 939, 938, 263, 1, 171, 937, 267, 981, 936, 173, + /* 660 */ 935, 934, 933, 932, 931, 930, 929, 873, 184, 272, + /* 670 */ 182, 928, 70, 185, 927, 187, 188, 8, 926, 924, + /* 680 */ 922, 920, 296, 194, 919, 917, 918, 767, 197, 28, + /* 690 */ 2, 913, 278, 96, 793, 98, 100, 804, 99, 237, + /* 700 */ 798, 102, 33, 800, 9, 103, 10, 290, 34, 298, + /* 710 */ 11, 119, 25, 26, 122, 36, 126, 654, 693, 692, + /* 720 */ 125, 309, 37, 128, 689, 687, 686, 685, 683, 682, + /* 730 */ 681, 678, 644, 132, 134, 135, 5, 328, 320, 848, + /* 740 */ 6, 329, 68, 40, 69, 725, 136, 724, 138, 143, + /* 750 */ 145, 721, 670, 668, 850, 660, 666, 662, 664, 658, + /* 760 */ 656, 691, 690, 688, 684, 680, 679, 608, 161, 642, + /* 770 */ 897, 896, 896, 896, 896, 896, 896, 896, 896, 191, /* 780 */ 192, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 194, 1, 239, 194, 3, 200, 243, 7, 8, 1, - /* 10 */ 10, 11, 192, 193, 14, 15, 16, 17, 241, 19, - /* 20 */ 20, 21, 22, 23, 24, 1, 1, 194, 195, 29, - /* 30 */ 30, 262, 200, 33, 34, 35, 259, 7, 8, 239, - /* 40 */ 10, 11, 273, 243, 14, 15, 16, 17, 194, 19, - /* 50 */ 20, 21, 22, 23, 24, 216, 262, 218, 219, 29, - /* 60 */ 30, 241, 223, 33, 34, 35, 227, 262, 229, 230, - /* 70 */ 264, 265, 263, 7, 8, 194, 10, 11, 273, 259, + /* 0 */ 195, 1, 240, 195, 3, 201, 244, 7, 8, 1, + /* 10 */ 10, 11, 193, 194, 14, 15, 16, 17, 242, 19, + /* 20 */ 20, 21, 22, 23, 24, 1, 1, 195, 196, 29, + /* 30 */ 30, 263, 201, 33, 34, 35, 260, 7, 8, 240, + /* 40 */ 10, 11, 274, 244, 14, 15, 16, 17, 195, 19, + /* 50 */ 20, 21, 22, 23, 24, 217, 263, 219, 220, 29, + /* 60 */ 30, 242, 224, 33, 34, 35, 228, 263, 230, 231, + /* 70 */ 265, 266, 264, 7, 8, 263, 10, 11, 274, 260, /* 80 */ 14, 15, 16, 17, 84, 19, 20, 21, 22, 23, - /* 90 */ 24, 194, 244, 245, 262, 29, 30, 214, 215, 33, - /* 100 */ 34, 35, 78, 7, 8, 273, 10, 11, 83, 79, - /* 110 */ 14, 15, 16, 17, 241, 19, 20, 21, 22, 23, - /* 120 */ 24, 240, 268, 78, 270, 29, 30, 124, 125, 33, - /* 130 */ 34, 35, 259, 201, 8, 238, 10, 11, 63, 242, - /* 140 */ 14, 15, 16, 17, 194, 19, 20, 21, 22, 23, - /* 150 */ 24, 96, 33, 98, 99, 29, 30, 1, 103, 33, - /* 160 */ 34, 35, 107, 118, 109, 110, 234, 235, 236, 237, - /* 170 */ 200, 96, 97, 98, 99, 100, 101, 102, 103, 104, - /* 180 */ 105, 106, 107, 108, 109, 110, 111, 216, 217, 218, - /* 190 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 200 */ 229, 230, 231, 232, 1, 10, 11, 74, 95, 14, - /* 210 */ 15, 16, 17, 249, 19, 20, 21, 22, 23, 24, - /* 220 */ 270, 88, 272, 88, 29, 30, 3, 4, 33, 34, - /* 230 */ 35, 267, 262, 29, 30, 122, 80, 33, 34, 35, - /* 240 */ 121, 3, 40, 273, 41, 42, 43, 44, 45, 46, + /* 90 */ 24, 195, 245, 246, 263, 29, 30, 215, 216, 33, + /* 100 */ 34, 35, 78, 7, 8, 274, 10, 11, 83, 79, + /* 110 */ 14, 15, 16, 17, 242, 19, 20, 21, 22, 23, + /* 120 */ 24, 202, 269, 78, 271, 29, 30, 124, 125, 33, + /* 130 */ 34, 35, 260, 202, 8, 239, 10, 11, 63, 243, + /* 140 */ 14, 15, 16, 17, 195, 19, 20, 21, 22, 23, + /* 150 */ 24, 96, 1, 98, 99, 29, 30, 238, 103, 33, + /* 160 */ 34, 35, 107, 118, 109, 110, 235, 236, 237, 238, + /* 170 */ 201, 96, 97, 98, 99, 100, 101, 102, 103, 104, + /* 180 */ 105, 106, 107, 108, 109, 110, 111, 217, 218, 219, + /* 190 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + /* 200 */ 230, 231, 232, 233, 1, 10, 11, 74, 250, 14, + /* 210 */ 15, 16, 17, 88, 19, 20, 21, 22, 23, 24, + /* 220 */ 271, 88, 273, 195, 29, 30, 268, 78, 33, 34, + /* 230 */ 35, 80, 263, 29, 30, 200, 202, 33, 34, 35, + /* 240 */ 33, 206, 40, 274, 41, 42, 43, 44, 45, 46, /* 250 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 57, - /* 260 */ 57, 1, 2, 3, 4, 5, 64, 29, 30, 58, - /* 270 */ 59, 60, 70, 71, 72, 73, 78, 80, 246, 77, + /* 260 */ 57, 1, 2, 3, 4, 5, 64, 118, 195, 120, + /* 270 */ 236, 243, 70, 71, 72, 73, 79, 204, 3, 77, /* 280 */ 78, 1, 2, 3, 4, 5, 33, 34, 35, 29, - /* 290 */ 30, 201, 260, 33, 21, 22, 23, 24, 2, 3, - /* 300 */ 4, 5, 29, 30, 3, 4, 33, 34, 35, 29, - /* 310 */ 30, 64, 2, 3, 4, 5, 118, 245, 120, 262, - /* 320 */ 118, 58, 59, 60, 194, 29, 30, 237, 194, 66, - /* 330 */ 67, 68, 69, 194, 194, 199, 194, 194, 78, 29, - /* 340 */ 30, 205, 140, 194, 142, 194, 123, 3, 4, 86, + /* 290 */ 30, 80, 95, 33, 21, 22, 23, 24, 2, 3, + /* 300 */ 4, 5, 29, 30, 29, 30, 33, 34, 35, 29, + /* 310 */ 30, 64, 2, 3, 4, 5, 3, 4, 3, 4, + /* 320 */ 118, 58, 59, 60, 263, 29, 30, 195, 121, 66, + /* 330 */ 67, 68, 69, 195, 195, 263, 195, 195, 78, 29, + /* 340 */ 30, 195, 140, 195, 142, 195, 3, 4, 195, 86, /* 350 */ 64, 149, 58, 59, 60, 58, 59, 60, 78, 65, - /* 360 */ 66, 67, 68, 69, 67, 68, 69, 194, 238, 111, - /* 370 */ 194, 201, 242, 76, 262, 194, 116, 117, 238, 262, - /* 380 */ 238, 238, 242, 123, 242, 242, 72, 238, 194, 238, - /* 390 */ 143, 242, 145, 242, 147, 148, 116, 117, 9, 141, - /* 400 */ 78, 194, 194, 123, 270, 235, 84, 194, 150, 270, - /* 410 */ 203, 238, 116, 117, 238, 242, 203, 79, 242, 238, - /* 420 */ 61, 62, 63, 242, 123, 79, 116, 117, 95, 143, - /* 430 */ 78, 145, 238, 147, 148, 79, 242, 115, 199, 194, - /* 440 */ 79, 199, 79, 79, 205, 95, 132, 205, 203, 1, - /* 450 */ 242, 95, 79, 79, 3, 4, 95, 119, 95, 95, - /* 460 */ 79, 79, 79, 79, 79, 119, 114, 56, 95, 95, - /* 470 */ 243, 138, 79, 144, 78, 146, 95, 95, 95, 95, - /* 480 */ 95, 33, 262, 144, 262, 146, 136, 262, 95, 78, - /* 490 */ 262, 144, 144, 146, 146, 3, 4, 3, 4, 144, - /* 500 */ 233, 146, 144, 262, 146, 116, 74, 75, 262, 113, - /* 510 */ 262, 262, 262, 262, 262, 245, 262, 262, 262, 262, - /* 520 */ 245, 262, 241, 262, 262, 233, 233, 261, 233, 241, - /* 530 */ 241, 233, 233, 233, 247, 271, 271, 194, 56, 194, - /* 540 */ 123, 135, 266, 241, 266, 196, 137, 194, 194, 266, - /* 550 */ 266, 194, 130, 134, 133, 128, 196, 254, 196, 194, - /* 560 */ 194, 194, 194, 258, 194, 196, 194, 196, 257, 256, - /* 570 */ 255, 253, 127, 252, 251, 126, 129, 250, 139, 248, - /* 580 */ 87, 194, 196, 204, 194, 94, 194, 196, 213, 194, - /* 590 */ 196, 194, 194, 194, 194, 93, 194, 194, 194, 47, - /* 600 */ 212, 194, 194, 90, 194, 204, 201, 194, 194, 194, - /* 610 */ 194, 92, 194, 51, 194, 91, 89, 80, 194, 196, - /* 620 */ 194, 196, 194, 194, 194, 194, 3, 194, 194, 3, - /* 630 */ 151, 3, 3, 3, 98, 97, 194, 194, 78, 119, - /* 640 */ 194, 78, 114, 207, 206, 141, 211, 210, 208, 194, - /* 650 */ 209, 194, 194, 194, 194, 194, 194, 194, 202, 215, - /* 660 */ 196, 194, 196, 194, 194, 194, 194, 194, 194, 194, - /* 670 */ 79, 194, 151, 197, 95, 197, 196, 194, 197, 194, - /* 680 */ 196, 194, 194, 197, 194, 196, 79, 95, 78, 194, - /* 690 */ 194, 194, 79, 194, 196, 194, 78, 95, 198, 79, - /* 700 */ 78, 1, 131, 79, 95, 78, 95, 131, 114, 78, - /* 710 */ 78, 78, 78, 115, 74, 66, 3, 3, 112, 3, - /* 720 */ 85, 84, 5, 85, 3, 84, 3, 3, 3, 3, - /* 730 */ 3, 3, 81, 74, 82, 82, 78, 116, 9, 79, - /* 740 */ 78, 55, 20, 10, 10, 146, 146, 146, 3, 95, - /* 750 */ 146, 3, 79, 3, 3, 3, 3, 3, 3, 3, - /* 760 */ 3, 3, 3, 3, 3, 3, 3, 95, 81, 0, - /* 770 */ 274, 56, 274, 274, 274, 274, 274, 274, 274, 15, - /* 780 */ 15, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 790 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 800 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 810 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 820 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 830 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 840 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 850 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 860 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 870 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 880 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 890 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 900 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 910 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 920 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 930 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 940 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 950 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 960 */ 274, 274, 274, 274, 274, 274, 274, 274, 274, 274, - /* 970 */ 274, 274, 274, + /* 360 */ 66, 67, 68, 69, 67, 68, 69, 195, 111, 78, + /* 370 */ 195, 239, 195, 76, 95, 243, 116, 117, 239, 241, + /* 380 */ 239, 239, 243, 123, 243, 243, 72, 239, 195, 239, + /* 390 */ 143, 243, 145, 243, 147, 148, 116, 117, 141, 200, + /* 400 */ 78, 122, 9, 123, 113, 206, 84, 150, 61, 62, + /* 410 */ 63, 239, 116, 117, 239, 243, 239, 271, 243, 200, + /* 420 */ 243, 58, 59, 60, 271, 206, 116, 117, 247, 143, + /* 430 */ 195, 145, 239, 147, 148, 195, 243, 115, 123, 204, + /* 440 */ 79, 79, 261, 79, 204, 79, 132, 79, 95, 95, + /* 450 */ 1, 79, 79, 79, 79, 3, 4, 79, 78, 95, + /* 460 */ 79, 95, 79, 95, 56, 242, 123, 95, 95, 95, + /* 470 */ 95, 79, 144, 95, 146, 144, 95, 146, 95, 263, + /* 480 */ 119, 119, 33, 144, 263, 146, 78, 95, 74, 75, + /* 490 */ 136, 138, 263, 263, 114, 3, 4, 263, 3, 4, + /* 500 */ 144, 144, 146, 146, 144, 263, 146, 263, 263, 116, + /* 510 */ 246, 263, 263, 263, 263, 263, 244, 263, 263, 263, + /* 520 */ 246, 263, 263, 246, 234, 242, 234, 234, 262, 234, + /* 530 */ 242, 242, 234, 234, 234, 248, 195, 272, 56, 272, + /* 540 */ 195, 197, 123, 195, 195, 195, 135, 195, 197, 267, + /* 550 */ 195, 267, 267, 195, 137, 130, 197, 134, 128, 256, + /* 560 */ 195, 259, 258, 257, 197, 195, 195, 133, 267, 127, + /* 570 */ 253, 255, 254, 197, 126, 252, 129, 251, 139, 249, + /* 580 */ 195, 197, 195, 197, 195, 197, 195, 87, 94, 195, + /* 590 */ 195, 195, 195, 214, 195, 205, 195, 195, 195, 195, + /* 600 */ 195, 195, 202, 205, 195, 195, 93, 195, 195, 47, + /* 610 */ 195, 90, 92, 197, 51, 195, 195, 195, 91, 195, + /* 620 */ 213, 195, 195, 89, 195, 197, 195, 80, 3, 3, + /* 630 */ 212, 3, 151, 195, 3, 195, 3, 98, 195, 195, + /* 640 */ 208, 195, 207, 198, 211, 209, 195, 210, 195, 195, + /* 650 */ 195, 195, 195, 203, 195, 197, 151, 216, 197, 195, + /* 660 */ 195, 195, 195, 195, 195, 195, 195, 97, 195, 141, + /* 670 */ 198, 197, 119, 198, 197, 195, 198, 78, 195, 195, + /* 680 */ 195, 195, 114, 195, 197, 195, 197, 79, 195, 78, + /* 690 */ 199, 195, 95, 95, 79, 78, 95, 79, 78, 1, + /* 700 */ 79, 78, 95, 79, 131, 78, 131, 78, 95, 114, + /* 710 */ 78, 115, 78, 78, 74, 85, 66, 3, 3, 3, + /* 720 */ 84, 112, 85, 84, 5, 3, 3, 3, 3, 3, + /* 730 */ 3, 3, 81, 74, 82, 82, 78, 20, 9, 79, + /* 740 */ 78, 55, 10, 95, 10, 3, 146, 3, 146, 146, + /* 750 */ 146, 79, 3, 3, 116, 3, 3, 3, 3, 3, + /* 760 */ 3, 3, 3, 3, 3, 3, 3, 56, 95, 81, + /* 770 */ 0, 275, 275, 275, 275, 275, 275, 275, 275, 15, + /* 780 */ 15, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 790 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 800 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 810 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 820 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 830 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 840 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 850 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 860 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 870 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 880 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 890 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 900 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 910 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 920 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 930 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 940 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 950 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 960 */ 275, 275, 275, 275, 275, 275, 275, 275, 275, 275, + /* 970 */ 275, 275, 275, }; #define YY_SHIFT_COUNT (367) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (769) +#define YY_SHIFT_MAX (770) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 202, 75, 55, 197, 260, 280, 280, 24, 8, 8, + /* 0 */ 202, 75, 55, 211, 260, 280, 280, 24, 8, 8, /* 10 */ 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, /* 20 */ 8, 0, 203, 280, 296, 310, 310, 45, 45, 3, - /* 30 */ 156, 133, 197, 8, 8, 8, 8, 8, 135, 8, - /* 40 */ 8, 135, 1, 781, 280, 280, 280, 280, 280, 280, + /* 30 */ 151, 133, 211, 8, 8, 8, 8, 8, 125, 8, + /* 40 */ 8, 125, 1, 781, 280, 280, 280, 280, 280, 280, /* 50 */ 280, 280, 280, 280, 280, 280, 280, 280, 280, 280, /* 60 */ 280, 280, 280, 280, 280, 280, 296, 310, 296, 296, - /* 70 */ 198, 238, 238, 238, 238, 238, 238, 238, 119, 45, - /* 80 */ 45, 314, 314, 113, 45, 25, 8, 482, 8, 8, - /* 90 */ 8, 482, 8, 8, 8, 482, 8, 417, 417, 417, - /* 100 */ 417, 482, 8, 8, 482, 406, 409, 422, 419, 421, - /* 110 */ 427, 445, 449, 447, 439, 482, 8, 8, 482, 8, - /* 120 */ 482, 8, 8, 8, 493, 8, 8, 493, 8, 8, - /* 130 */ 8, 197, 8, 8, 8, 8, 8, 8, 8, 8, + /* 70 */ 149, 275, 275, 275, 275, 275, 275, 275, 207, 45, + /* 80 */ 45, 314, 314, 279, 45, 25, 8, 482, 8, 8, + /* 90 */ 8, 482, 8, 8, 8, 482, 8, 419, 419, 419, + /* 100 */ 419, 482, 8, 8, 482, 411, 417, 425, 423, 434, + /* 110 */ 430, 442, 448, 447, 439, 482, 8, 8, 482, 8, + /* 120 */ 482, 8, 8, 8, 500, 8, 8, 500, 8, 8, + /* 130 */ 8, 211, 8, 8, 8, 8, 8, 8, 8, 8, /* 140 */ 8, 482, 8, 8, 8, 8, 8, 8, 482, 8, - /* 150 */ 8, 491, 502, 552, 513, 519, 562, 524, 527, 8, + /* 150 */ 8, 494, 513, 562, 521, 520, 563, 527, 534, 8, /* 160 */ 8, 1, 8, 8, 8, 8, 8, 8, 8, 8, /* 170 */ 8, 482, 8, 482, 8, 8, 8, 8, 8, 8, - /* 180 */ 8, 537, 8, 537, 482, 8, 537, 482, 8, 537, + /* 180 */ 8, 547, 8, 547, 482, 8, 547, 482, 8, 547, /* 190 */ 8, 8, 8, 8, 482, 8, 8, 482, 8, 8, /* 200 */ 781, 781, 30, 66, 66, 96, 66, 126, 195, 273, /* 210 */ 273, 273, 273, 273, 273, 263, 294, 297, 204, 204, - /* 220 */ 204, 204, 247, 286, 258, 322, 253, 253, 223, 301, - /* 230 */ 359, 211, 356, 338, 346, 361, 363, 364, 333, 350, - /* 240 */ 373, 374, 381, 382, 344, 451, 383, 352, 384, 385, - /* 250 */ 448, 411, 389, 393, 329, 339, 347, 492, 494, 348, - /* 260 */ 355, 396, 358, 432, 623, 479, 626, 628, 521, 629, - /* 270 */ 630, 536, 538, 504, 520, 528, 560, 591, 563, 579, - /* 280 */ 592, 607, 610, 613, 618, 620, 602, 622, 624, 627, - /* 290 */ 700, 631, 609, 571, 611, 576, 632, 528, 633, 594, - /* 300 */ 634, 598, 640, 635, 637, 649, 713, 638, 641, 714, - /* 310 */ 606, 716, 717, 721, 723, 724, 725, 726, 727, 728, - /* 320 */ 651, 729, 659, 652, 653, 658, 660, 621, 662, 722, - /* 330 */ 686, 733, 599, 600, 654, 654, 654, 654, 734, 601, - /* 340 */ 604, 654, 654, 654, 745, 748, 673, 654, 750, 751, + /* 220 */ 204, 204, 247, 286, 257, 322, 253, 253, 315, 343, + /* 230 */ 347, 363, 197, 361, 362, 364, 366, 368, 353, 354, + /* 240 */ 372, 373, 374, 375, 313, 452, 378, 380, 381, 383, + /* 250 */ 449, 408, 393, 392, 328, 331, 339, 492, 495, 356, + /* 260 */ 357, 291, 360, 414, 625, 481, 626, 628, 505, 631, + /* 270 */ 633, 539, 570, 528, 553, 568, 599, 608, 611, 597, + /* 280 */ 598, 615, 617, 618, 620, 621, 601, 623, 624, 627, + /* 290 */ 698, 629, 607, 573, 613, 575, 632, 568, 634, 595, + /* 300 */ 635, 596, 640, 630, 636, 650, 714, 637, 639, 715, + /* 310 */ 609, 716, 719, 722, 723, 724, 725, 726, 727, 728, + /* 320 */ 651, 729, 659, 652, 653, 658, 660, 638, 662, 717, + /* 330 */ 686, 732, 600, 602, 648, 648, 648, 648, 734, 603, + /* 340 */ 604, 648, 648, 648, 742, 744, 672, 648, 749, 750, /* 350 */ 752, 753, 754, 755, 756, 757, 758, 759, 760, 761, - /* 360 */ 762, 763, 672, 687, 764, 765, 715, 769, + /* 360 */ 762, 763, 673, 688, 764, 765, 711, 770, }; #define YY_REDUCE_COUNT (201) -#define YY_REDUCE_MIN (-237) -#define YY_REDUCE_MAX (501) +#define YY_REDUCE_MIN (-238) +#define YY_REDUCE_MAX (496) static const short yy_reduce_ofst[] = { - /* 0 */ -180, -29, -161, -68, -195, -168, -30, -194, -103, -50, - /* 10 */ -146, 130, 140, 142, 143, 149, 151, 173, 176, 181, - /* 20 */ 194, -191, -167, -231, -152, -237, -200, -223, -127, -36, - /* 30 */ 90, 136, 170, 134, 139, -119, 207, 213, 239, 245, - /* 40 */ 208, 242, -117, 32, -206, 57, 112, 117, 220, 222, - /* 50 */ 225, 228, 241, 246, 248, 249, 250, 251, 252, 254, - /* 60 */ 255, 256, 257, 259, 261, 262, 72, 227, 270, 275, - /* 70 */ 281, 267, 292, 293, 295, 298, 299, 300, 266, 288, - /* 80 */ 289, 264, 265, 287, 302, 343, 345, 349, 353, 354, - /* 90 */ 357, 360, 365, 366, 367, 362, 368, 276, 278, 283, - /* 100 */ 284, 369, 370, 372, 371, 305, 311, 313, 315, 303, - /* 110 */ 318, 321, 323, 327, 331, 386, 387, 390, 391, 392, - /* 120 */ 394, 395, 397, 398, 379, 399, 400, 401, 402, 403, - /* 130 */ 404, 405, 407, 408, 410, 413, 414, 415, 416, 418, - /* 140 */ 420, 423, 424, 426, 428, 429, 430, 431, 425, 433, - /* 150 */ 434, 375, 388, 435, 436, 437, 440, 441, 438, 442, - /* 160 */ 443, 444, 446, 455, 457, 458, 459, 460, 461, 462, - /* 170 */ 463, 464, 467, 466, 469, 470, 471, 472, 473, 474, - /* 180 */ 475, 476, 477, 478, 480, 483, 481, 484, 485, 486, - /* 190 */ 487, 488, 490, 495, 489, 496, 497, 498, 499, 501, - /* 200 */ 456, 500, + /* 0 */ -181, -30, -162, -69, -196, -169, -31, -195, -104, -51, + /* 10 */ -147, 132, 139, 141, 142, 148, 150, 172, 175, 177, + /* 20 */ 193, -192, -168, -232, -153, -238, -201, -224, -128, -42, + /* 30 */ -81, 35, 34, 146, 153, 138, 73, 235, 199, 240, + /* 40 */ 28, 219, -118, 181, -207, -188, 61, 72, 216, 221, + /* 50 */ 229, 230, 234, 242, 244, 245, 248, 249, 250, 251, + /* 60 */ 252, 254, 255, 256, 258, 259, 264, 272, 274, 277, + /* 70 */ 223, 290, 292, 293, 295, 298, 299, 300, 266, 283, + /* 80 */ 288, 265, 267, 287, 289, 341, 345, 344, 348, 349, + /* 90 */ 350, 351, 352, 355, 358, 359, 365, 282, 284, 285, + /* 100 */ 301, 367, 370, 371, 376, 302, 304, 306, 303, 316, + /* 110 */ 318, 317, 323, 326, 330, 384, 385, 387, 386, 389, + /* 120 */ 388, 391, 394, 395, 390, 396, 397, 398, 399, 401, + /* 130 */ 402, 400, 403, 404, 405, 406, 409, 410, 412, 413, + /* 140 */ 415, 416, 420, 421, 422, 424, 426, 427, 428, 429, + /* 150 */ 431, 379, 407, 418, 432, 433, 436, 437, 435, 438, + /* 160 */ 440, 441, 443, 444, 446, 451, 453, 454, 455, 456, + /* 170 */ 457, 458, 459, 461, 464, 465, 466, 467, 468, 469, + /* 180 */ 470, 445, 471, 472, 474, 473, 475, 477, 480, 478, + /* 190 */ 483, 484, 485, 486, 487, 488, 490, 489, 493, 496, + /* 200 */ 450, 491, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 894, 956, 944, 952, 1179, 1179, 1179, 894, 894, 894, @@ -754,7 +744,6 @@ struct yyParser { int yyerrcnt; /* Shifts left before out of the error */ #endif ParseARG_SDECL /* A place to hold %extra_argument */ - ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ @@ -994,88 +983,89 @@ static const char *const yyTokenName[] = { /* 189 */ "INSERT", /* 190 */ "INTO", /* 191 */ "VALUES", - /* 192 */ "program", - /* 193 */ "cmd", - /* 194 */ "ids", - /* 195 */ "dbPrefix", - /* 196 */ "cpxName", - /* 197 */ "ifexists", - /* 198 */ "alter_db_optr", - /* 199 */ "acct_optr", - /* 200 */ "exprlist", - /* 201 */ "ifnotexists", - /* 202 */ "db_optr", - /* 203 */ "typename", - /* 204 */ "bufsize", - /* 205 */ "pps", - /* 206 */ "tseries", - /* 207 */ "dbs", - /* 208 */ "streams", - /* 209 */ "storage", - /* 210 */ "qtime", - /* 211 */ "users", - /* 212 */ "conns", - /* 213 */ "state", - /* 214 */ "intitemlist", - /* 215 */ "intitem", - /* 216 */ "keep", - /* 217 */ "cache", - /* 218 */ "replica", - /* 219 */ "quorum", - /* 220 */ "days", - /* 221 */ "minrows", - /* 222 */ "maxrows", - /* 223 */ "blocks", - /* 224 */ "ctime", - /* 225 */ "wal", - /* 226 */ "fsync", - /* 227 */ "comp", - /* 228 */ "prec", - /* 229 */ "update", - /* 230 */ "cachelast", - /* 231 */ "vgroups", - /* 232 */ "stream_mode", - /* 233 */ "signed", - /* 234 */ "create_table_args", - /* 235 */ "create_stable_args", - /* 236 */ "create_table_list", - /* 237 */ "create_from_stable", - /* 238 */ "columnlist", - /* 239 */ "tagitemlist1", - /* 240 */ "tagNamelist", - /* 241 */ "select", - /* 242 */ "column", - /* 243 */ "tagitem1", - /* 244 */ "tagitemlist", - /* 245 */ "tagitem", - /* 246 */ "selcollist", - /* 247 */ "from", - /* 248 */ "where_opt", - /* 249 */ "interval_option", - /* 250 */ "sliding_opt", - /* 251 */ "session_option", - /* 252 */ "windowstate_option", - /* 253 */ "fill_opt", - /* 254 */ "groupby_opt", - /* 255 */ "having_opt", - /* 256 */ "orderby_opt", - /* 257 */ "slimit_opt", - /* 258 */ "limit_opt", - /* 259 */ "union", - /* 260 */ "sclp", - /* 261 */ "distinct", - /* 262 */ "expr", - /* 263 */ "as", - /* 264 */ "tablelist", - /* 265 */ "sub", - /* 266 */ "tmvar", - /* 267 */ "intervalKey", - /* 268 */ "sortlist", - /* 269 */ "sortitem", - /* 270 */ "item", - /* 271 */ "sortorder", - /* 272 */ "grouplist", - /* 273 */ "expritem", + /* 192 */ "error", + /* 193 */ "program", + /* 194 */ "cmd", + /* 195 */ "ids", + /* 196 */ "dbPrefix", + /* 197 */ "cpxName", + /* 198 */ "ifexists", + /* 199 */ "alter_db_optr", + /* 200 */ "acct_optr", + /* 201 */ "exprlist", + /* 202 */ "ifnotexists", + /* 203 */ "db_optr", + /* 204 */ "typename", + /* 205 */ "bufsize", + /* 206 */ "pps", + /* 207 */ "tseries", + /* 208 */ "dbs", + /* 209 */ "streams", + /* 210 */ "storage", + /* 211 */ "qtime", + /* 212 */ "users", + /* 213 */ "conns", + /* 214 */ "state", + /* 215 */ "intitemlist", + /* 216 */ "intitem", + /* 217 */ "keep", + /* 218 */ "cache", + /* 219 */ "replica", + /* 220 */ "quorum", + /* 221 */ "days", + /* 222 */ "minrows", + /* 223 */ "maxrows", + /* 224 */ "blocks", + /* 225 */ "ctime", + /* 226 */ "wal", + /* 227 */ "fsync", + /* 228 */ "comp", + /* 229 */ "prec", + /* 230 */ "update", + /* 231 */ "cachelast", + /* 232 */ "vgroups", + /* 233 */ "stream_mode", + /* 234 */ "signed", + /* 235 */ "create_table_args", + /* 236 */ "create_stable_args", + /* 237 */ "create_table_list", + /* 238 */ "create_from_stable", + /* 239 */ "columnlist", + /* 240 */ "tagitemlist1", + /* 241 */ "tagNamelist", + /* 242 */ "select", + /* 243 */ "column", + /* 244 */ "tagitem1", + /* 245 */ "tagitemlist", + /* 246 */ "tagitem", + /* 247 */ "selcollist", + /* 248 */ "from", + /* 249 */ "where_opt", + /* 250 */ "interval_option", + /* 251 */ "sliding_opt", + /* 252 */ "session_option", + /* 253 */ "windowstate_option", + /* 254 */ "fill_opt", + /* 255 */ "groupby_opt", + /* 256 */ "having_opt", + /* 257 */ "orderby_opt", + /* 258 */ "slimit_opt", + /* 259 */ "limit_opt", + /* 260 */ "union", + /* 261 */ "sclp", + /* 262 */ "distinct", + /* 263 */ "expr", + /* 264 */ "as", + /* 265 */ "tablelist", + /* 266 */ "sub", + /* 267 */ "tmvar", + /* 268 */ "intervalKey", + /* 269 */ "sortlist", + /* 270 */ "sortitem", + /* 271 */ "item", + /* 272 */ "sortorder", + /* 273 */ "grouplist", + /* 274 */ "expritem", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1435,29 +1425,28 @@ static int yyGrowStack(yyParser *p){ /* Initialize a new parser that has already been allocated. */ -void ParseInit(void *yypRawParser ParseCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - ParseCTX_STORE +void ParseInit(void *yypParser){ + yyParser *pParser = (yyParser*)yypParser; #ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; + pParser->yyhwm = 0; #endif #if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; + pParser->yytos = NULL; + pParser->yystack = NULL; + pParser->yystksz = 0; + if( yyGrowStack(pParser) ){ + pParser->yystack = &pParser->yystk0; + pParser->yystksz = 1; } #endif #ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; + pParser->yyerrcnt = -1; #endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; + pParser->yytos = pParser->yystack; + pParser->yystack[0].stateno = 0; + pParser->yystack[0].major = 0; #if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; + pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1]; #endif } @@ -1474,14 +1463,11 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - ParseCTX_STORE - ParseInit(yypParser ParseCTX_PARAM); - } - return (void*)yypParser; +void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ + yyParser *pParser; + pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( pParser ) ParseInit(pParser); + return pParser; } #endif /* Parse_ENGINEALWAYSONSTACK */ @@ -1498,8 +1484,7 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1512,61 +1497,61 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 200: /* exprlist */ - case 246: /* selcollist */ - case 260: /* sclp */ + case 201: /* exprlist */ + case 247: /* selcollist */ + case 261: /* sclp */ { -tSqlExprListDestroy((yypminor->yy225)); +tSqlExprListDestroy((yypminor->yy131)); } break; - case 214: /* intitemlist */ - case 216: /* keep */ - case 238: /* columnlist */ - case 239: /* tagitemlist1 */ - case 240: /* tagNamelist */ - case 244: /* tagitemlist */ - case 253: /* fill_opt */ - case 254: /* groupby_opt */ - case 256: /* orderby_opt */ - case 268: /* sortlist */ - case 272: /* grouplist */ -{ -taosArrayDestroy((yypminor->yy225)); + case 215: /* intitemlist */ + case 217: /* keep */ + case 239: /* columnlist */ + case 240: /* tagitemlist1 */ + case 241: /* tagNamelist */ + case 245: /* tagitemlist */ + case 254: /* fill_opt */ + case 255: /* groupby_opt */ + case 257: /* orderby_opt */ + case 269: /* sortlist */ + case 273: /* grouplist */ +{ +taosArrayDestroy((yypminor->yy131)); } break; - case 236: /* create_table_list */ + case 237: /* create_table_list */ { -destroyCreateTableSql((yypminor->yy482)); +destroyCreateTableSql((yypminor->yy272)); } break; - case 241: /* select */ + case 242: /* select */ { -destroySqlNode((yypminor->yy185)); +destroySqlNode((yypminor->yy256)); } break; - case 247: /* from */ - case 264: /* tablelist */ - case 265: /* sub */ + case 248: /* from */ + case 265: /* tablelist */ + case 266: /* sub */ { -destroyRelationInfo((yypminor->yy160)); +destroyRelationInfo((yypminor->yy544)); } break; - case 248: /* where_opt */ - case 255: /* having_opt */ - case 262: /* expr */ - case 273: /* expritem */ + case 249: /* where_opt */ + case 256: /* having_opt */ + case 263: /* expr */ + case 274: /* expritem */ { -tSqlExprDestroy((yypminor->yy226)); +tSqlExprDestroy((yypminor->yy46)); } break; - case 259: /* union */ + case 260: /* union */ { -destroyAllSqlNode((yypminor->yy93)); +destroyAllSqlNode((yypminor->yy303)); } break; - case 269: /* sortitem */ + case 270: /* sortitem */ { -taosVariantDestroy(&(yypminor->yy1)); +taosVariantDestroy(&(yypminor->yy43)); } break; /********* End destructor definitions *****************************************/ @@ -1678,12 +1663,13 @@ int ParseCoverage(FILE *out){ ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. */ -static YYACTIONTYPE yy_find_shift_action( - YYCODETYPE iLookAhead, /* The look-ahead token */ - YYACTIONTYPE stateno /* Current state number */ +static unsigned int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; - + int stateno = pParser->yytos->stateno; + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); #if defined(YYCOVERAGE) @@ -1691,12 +1677,11 @@ static YYACTIONTYPE yy_find_shift_action( #endif do{ i = yy_shift_ofst[stateno]; - assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i>=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ if( iLookAhead=YY_ACTTAB_COUNT j0 ){ #ifndef NDEBUG @@ -1747,8 +1731,8 @@ static YYACTIONTYPE yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static YYACTIONTYPE yy_find_reduce_action( - YYACTIONTYPE stateno, /* Current state number */ +static int yy_find_reduce_action( + int stateno, /* Current state number */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; @@ -1777,8 +1761,7 @@ static YYACTIONTYPE yy_find_reduce_action( ** The following routine is called if the stack overflows. */ static void yyStackOverflow(yyParser *yypParser){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); @@ -1789,8 +1772,7 @@ static void yyStackOverflow(yyParser *yypParser){ ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ /******** End %stack_overflow code ********************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument var */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } /* @@ -1819,8 +1801,8 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ ParseTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; @@ -1850,8 +1832,8 @@ static void yy_shift( yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; } yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; yytos->minor.yy0 = yyMinor; yyTraceShift(yypParser, yyNewState, "Shift"); } @@ -1863,310 +1845,310 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 192, -1 }, /* (0) program ::= cmd */ - { 193, -2 }, /* (1) cmd ::= SHOW DATABASES */ - { 193, -2 }, /* (2) cmd ::= SHOW TOPICS */ - { 193, -2 }, /* (3) cmd ::= SHOW FUNCTIONS */ - { 193, -2 }, /* (4) cmd ::= SHOW MNODES */ - { 193, -2 }, /* (5) cmd ::= SHOW DNODES */ - { 193, -2 }, /* (6) cmd ::= SHOW ACCOUNTS */ - { 193, -2 }, /* (7) cmd ::= SHOW USERS */ - { 193, -2 }, /* (8) cmd ::= SHOW MODULES */ - { 193, -2 }, /* (9) cmd ::= SHOW QUERIES */ - { 193, -2 }, /* (10) cmd ::= SHOW CONNECTIONS */ - { 193, -2 }, /* (11) cmd ::= SHOW STREAMS */ - { 193, -2 }, /* (12) cmd ::= SHOW VARIABLES */ - { 193, -2 }, /* (13) cmd ::= SHOW SCORES */ - { 193, -2 }, /* (14) cmd ::= SHOW GRANTS */ - { 193, -2 }, /* (15) cmd ::= SHOW VNODES */ - { 193, -3 }, /* (16) cmd ::= SHOW VNODES ids */ - { 195, 0 }, /* (17) dbPrefix ::= */ - { 195, -2 }, /* (18) dbPrefix ::= ids DOT */ - { 196, 0 }, /* (19) cpxName ::= */ - { 196, -2 }, /* (20) cpxName ::= DOT ids */ - { 193, -5 }, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ - { 193, -5 }, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ - { 193, -4 }, /* (23) cmd ::= SHOW CREATE DATABASE ids */ - { 193, -3 }, /* (24) cmd ::= SHOW dbPrefix TABLES */ - { 193, -5 }, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - { 193, -3 }, /* (26) cmd ::= SHOW dbPrefix STABLES */ - { 193, -5 }, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - { 193, -3 }, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ - { 193, -4 }, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ - { 193, -5 }, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ - { 193, -5 }, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ - { 193, -4 }, /* (32) cmd ::= DROP DATABASE ifexists ids */ - { 193, -4 }, /* (33) cmd ::= DROP TOPIC ifexists ids */ - { 193, -3 }, /* (34) cmd ::= DROP FUNCTION ids */ - { 193, -3 }, /* (35) cmd ::= DROP DNODE ids */ - { 193, -3 }, /* (36) cmd ::= DROP USER ids */ - { 193, -3 }, /* (37) cmd ::= DROP ACCOUNT ids */ - { 193, -2 }, /* (38) cmd ::= USE ids */ - { 193, -3 }, /* (39) cmd ::= DESCRIBE ids cpxName */ - { 193, -3 }, /* (40) cmd ::= DESC ids cpxName */ - { 193, -5 }, /* (41) cmd ::= ALTER USER ids PASS ids */ - { 193, -5 }, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ - { 193, -4 }, /* (43) cmd ::= ALTER DNODE ids ids */ - { 193, -5 }, /* (44) cmd ::= ALTER DNODE ids ids ids */ - { 193, -3 }, /* (45) cmd ::= ALTER LOCAL ids */ - { 193, -4 }, /* (46) cmd ::= ALTER LOCAL ids ids */ - { 193, -4 }, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ - { 193, -4 }, /* (48) cmd ::= ALTER ACCOUNT ids acct_optr */ - { 193, -6 }, /* (49) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - { 193, -6 }, /* (50) cmd ::= COMPACT VNODES IN LP exprlist RP */ - { 194, -1 }, /* (51) ids ::= ID */ - { 197, -2 }, /* (52) ifexists ::= IF EXISTS */ - { 197, 0 }, /* (53) ifexists ::= */ - { 201, -3 }, /* (54) ifnotexists ::= IF NOT EXISTS */ - { 201, 0 }, /* (55) ifnotexists ::= */ - { 193, -5 }, /* (56) cmd ::= CREATE DNODE ids PORT ids */ - { 193, -5 }, /* (57) cmd ::= CREATE DNODE IPTOKEN PORT ids */ - { 193, -6 }, /* (58) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - { 193, -5 }, /* (59) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - { 193, -8 }, /* (60) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - { 193, -9 }, /* (61) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ - { 193, -5 }, /* (62) cmd ::= CREATE USER ids PASS ids */ - { 204, 0 }, /* (63) bufsize ::= */ - { 204, -2 }, /* (64) bufsize ::= BUFSIZE INTEGER */ - { 205, 0 }, /* (65) pps ::= */ - { 205, -2 }, /* (66) pps ::= PPS INTEGER */ - { 206, 0 }, /* (67) tseries ::= */ - { 206, -2 }, /* (68) tseries ::= TSERIES INTEGER */ - { 207, 0 }, /* (69) dbs ::= */ - { 207, -2 }, /* (70) dbs ::= DBS INTEGER */ - { 208, 0 }, /* (71) streams ::= */ - { 208, -2 }, /* (72) streams ::= STREAMS INTEGER */ - { 209, 0 }, /* (73) storage ::= */ - { 209, -2 }, /* (74) storage ::= STORAGE INTEGER */ - { 210, 0 }, /* (75) qtime ::= */ - { 210, -2 }, /* (76) qtime ::= QTIME INTEGER */ - { 211, 0 }, /* (77) users ::= */ - { 211, -2 }, /* (78) users ::= USERS INTEGER */ - { 212, 0 }, /* (79) conns ::= */ - { 212, -2 }, /* (80) conns ::= CONNS INTEGER */ - { 213, 0 }, /* (81) state ::= */ - { 213, -2 }, /* (82) state ::= STATE ids */ - { 199, -9 }, /* (83) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - { 214, -3 }, /* (84) intitemlist ::= intitemlist COMMA intitem */ - { 214, -1 }, /* (85) intitemlist ::= intitem */ - { 215, -1 }, /* (86) intitem ::= INTEGER */ - { 216, -2 }, /* (87) keep ::= KEEP intitemlist */ - { 217, -2 }, /* (88) cache ::= CACHE INTEGER */ - { 218, -2 }, /* (89) replica ::= REPLICA INTEGER */ - { 219, -2 }, /* (90) quorum ::= QUORUM INTEGER */ - { 220, -2 }, /* (91) days ::= DAYS INTEGER */ - { 221, -2 }, /* (92) minrows ::= MINROWS INTEGER */ - { 222, -2 }, /* (93) maxrows ::= MAXROWS INTEGER */ - { 223, -2 }, /* (94) blocks ::= BLOCKS INTEGER */ - { 224, -2 }, /* (95) ctime ::= CTIME INTEGER */ - { 225, -2 }, /* (96) wal ::= WAL INTEGER */ - { 226, -2 }, /* (97) fsync ::= FSYNC INTEGER */ - { 227, -2 }, /* (98) comp ::= COMP INTEGER */ - { 228, -2 }, /* (99) prec ::= PRECISION STRING */ - { 229, -2 }, /* (100) update ::= UPDATE INTEGER */ - { 230, -2 }, /* (101) cachelast ::= CACHELAST INTEGER */ - { 231, -2 }, /* (102) vgroups ::= VGROUPS INTEGER */ - { 232, -3 }, /* (103) stream_mode ::= STREAM MODE INTEGER */ - { 202, 0 }, /* (104) db_optr ::= */ - { 202, -2 }, /* (105) db_optr ::= db_optr cache */ - { 202, -2 }, /* (106) db_optr ::= db_optr replica */ - { 202, -2 }, /* (107) db_optr ::= db_optr quorum */ - { 202, -2 }, /* (108) db_optr ::= db_optr days */ - { 202, -2 }, /* (109) db_optr ::= db_optr minrows */ - { 202, -2 }, /* (110) db_optr ::= db_optr maxrows */ - { 202, -2 }, /* (111) db_optr ::= db_optr blocks */ - { 202, -2 }, /* (112) db_optr ::= db_optr ctime */ - { 202, -2 }, /* (113) db_optr ::= db_optr wal */ - { 202, -2 }, /* (114) db_optr ::= db_optr fsync */ - { 202, -2 }, /* (115) db_optr ::= db_optr comp */ - { 202, -2 }, /* (116) db_optr ::= db_optr prec */ - { 202, -2 }, /* (117) db_optr ::= db_optr keep */ - { 202, -2 }, /* (118) db_optr ::= db_optr update */ - { 202, -2 }, /* (119) db_optr ::= db_optr cachelast */ - { 202, -2 }, /* (120) db_optr ::= db_optr vgroups */ - { 202, -2 }, /* (121) db_optr ::= db_optr stream_mode */ - { 198, 0 }, /* (122) alter_db_optr ::= */ - { 198, -2 }, /* (123) alter_db_optr ::= alter_db_optr replica */ - { 198, -2 }, /* (124) alter_db_optr ::= alter_db_optr quorum */ - { 198, -2 }, /* (125) alter_db_optr ::= alter_db_optr keep */ - { 198, -2 }, /* (126) alter_db_optr ::= alter_db_optr blocks */ - { 198, -2 }, /* (127) alter_db_optr ::= alter_db_optr comp */ - { 198, -2 }, /* (128) alter_db_optr ::= alter_db_optr update */ - { 198, -2 }, /* (129) alter_db_optr ::= alter_db_optr cachelast */ - { 203, -1 }, /* (130) typename ::= ids */ - { 203, -4 }, /* (131) typename ::= ids LP signed RP */ - { 203, -2 }, /* (132) typename ::= ids UNSIGNED */ - { 233, -1 }, /* (133) signed ::= INTEGER */ - { 233, -2 }, /* (134) signed ::= PLUS INTEGER */ - { 233, -2 }, /* (135) signed ::= MINUS INTEGER */ - { 193, -3 }, /* (136) cmd ::= CREATE TABLE create_table_args */ - { 193, -3 }, /* (137) cmd ::= CREATE TABLE create_stable_args */ - { 193, -3 }, /* (138) cmd ::= CREATE STABLE create_stable_args */ - { 193, -3 }, /* (139) cmd ::= CREATE TABLE create_table_list */ - { 236, -1 }, /* (140) create_table_list ::= create_from_stable */ - { 236, -2 }, /* (141) create_table_list ::= create_table_list create_from_stable */ - { 234, -6 }, /* (142) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ - { 235, -10 }, /* (143) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ - { 237, -10 }, /* (144) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist1 RP */ - { 237, -13 }, /* (145) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist1 RP */ - { 240, -3 }, /* (146) tagNamelist ::= tagNamelist COMMA ids */ - { 240, -1 }, /* (147) tagNamelist ::= ids */ - { 234, -5 }, /* (148) create_table_args ::= ifnotexists ids cpxName AS select */ - { 238, -3 }, /* (149) columnlist ::= columnlist COMMA column */ - { 238, -1 }, /* (150) columnlist ::= column */ - { 242, -2 }, /* (151) column ::= ids typename */ - { 239, -3 }, /* (152) tagitemlist1 ::= tagitemlist1 COMMA tagitem1 */ - { 239, -1 }, /* (153) tagitemlist1 ::= tagitem1 */ - { 243, -2 }, /* (154) tagitem1 ::= MINUS INTEGER */ - { 243, -2 }, /* (155) tagitem1 ::= MINUS FLOAT */ - { 243, -2 }, /* (156) tagitem1 ::= PLUS INTEGER */ - { 243, -2 }, /* (157) tagitem1 ::= PLUS FLOAT */ - { 243, -1 }, /* (158) tagitem1 ::= INTEGER */ - { 243, -1 }, /* (159) tagitem1 ::= FLOAT */ - { 243, -1 }, /* (160) tagitem1 ::= STRING */ - { 243, -1 }, /* (161) tagitem1 ::= BOOL */ - { 243, -1 }, /* (162) tagitem1 ::= NULL */ - { 243, -1 }, /* (163) tagitem1 ::= NOW */ - { 244, -3 }, /* (164) tagitemlist ::= tagitemlist COMMA tagitem */ - { 244, -1 }, /* (165) tagitemlist ::= tagitem */ - { 245, -1 }, /* (166) tagitem ::= INTEGER */ - { 245, -1 }, /* (167) tagitem ::= FLOAT */ - { 245, -1 }, /* (168) tagitem ::= STRING */ - { 245, -1 }, /* (169) tagitem ::= BOOL */ - { 245, -1 }, /* (170) tagitem ::= NULL */ - { 245, -1 }, /* (171) tagitem ::= NOW */ - { 245, -2 }, /* (172) tagitem ::= MINUS INTEGER */ - { 245, -2 }, /* (173) tagitem ::= MINUS FLOAT */ - { 245, -2 }, /* (174) tagitem ::= PLUS INTEGER */ - { 245, -2 }, /* (175) tagitem ::= PLUS FLOAT */ - { 241, -14 }, /* (176) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ - { 241, -3 }, /* (177) select ::= LP select RP */ - { 259, -1 }, /* (178) union ::= select */ - { 259, -4 }, /* (179) union ::= union UNION ALL select */ - { 259, -3 }, /* (180) union ::= union UNION select */ - { 193, -1 }, /* (181) cmd ::= union */ - { 241, -2 }, /* (182) select ::= SELECT selcollist */ - { 260, -2 }, /* (183) sclp ::= selcollist COMMA */ - { 260, 0 }, /* (184) sclp ::= */ - { 246, -4 }, /* (185) selcollist ::= sclp distinct expr as */ - { 246, -2 }, /* (186) selcollist ::= sclp STAR */ - { 263, -2 }, /* (187) as ::= AS ids */ - { 263, -1 }, /* (188) as ::= ids */ - { 263, 0 }, /* (189) as ::= */ - { 261, -1 }, /* (190) distinct ::= DISTINCT */ - { 261, 0 }, /* (191) distinct ::= */ - { 247, -2 }, /* (192) from ::= FROM tablelist */ - { 247, -2 }, /* (193) from ::= FROM sub */ - { 265, -3 }, /* (194) sub ::= LP union RP */ - { 265, -4 }, /* (195) sub ::= LP union RP ids */ - { 265, -6 }, /* (196) sub ::= sub COMMA LP union RP ids */ - { 264, -2 }, /* (197) tablelist ::= ids cpxName */ - { 264, -3 }, /* (198) tablelist ::= ids cpxName ids */ - { 264, -4 }, /* (199) tablelist ::= tablelist COMMA ids cpxName */ - { 264, -5 }, /* (200) tablelist ::= tablelist COMMA ids cpxName ids */ - { 266, -1 }, /* (201) tmvar ::= VARIABLE */ - { 249, -4 }, /* (202) interval_option ::= intervalKey LP tmvar RP */ - { 249, -6 }, /* (203) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ - { 249, 0 }, /* (204) interval_option ::= */ - { 267, -1 }, /* (205) intervalKey ::= INTERVAL */ - { 267, -1 }, /* (206) intervalKey ::= EVERY */ - { 251, 0 }, /* (207) session_option ::= */ - { 251, -7 }, /* (208) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ - { 252, 0 }, /* (209) windowstate_option ::= */ - { 252, -4 }, /* (210) windowstate_option ::= STATE_WINDOW LP ids RP */ - { 253, 0 }, /* (211) fill_opt ::= */ - { 253, -6 }, /* (212) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 253, -4 }, /* (213) fill_opt ::= FILL LP ID RP */ - { 250, -4 }, /* (214) sliding_opt ::= SLIDING LP tmvar RP */ - { 250, 0 }, /* (215) sliding_opt ::= */ - { 256, 0 }, /* (216) orderby_opt ::= */ - { 256, -3 }, /* (217) orderby_opt ::= ORDER BY sortlist */ - { 268, -4 }, /* (218) sortlist ::= sortlist COMMA item sortorder */ - { 268, -2 }, /* (219) sortlist ::= item sortorder */ - { 270, -2 }, /* (220) item ::= ids cpxName */ - { 271, -1 }, /* (221) sortorder ::= ASC */ - { 271, -1 }, /* (222) sortorder ::= DESC */ - { 271, 0 }, /* (223) sortorder ::= */ - { 254, 0 }, /* (224) groupby_opt ::= */ - { 254, -3 }, /* (225) groupby_opt ::= GROUP BY grouplist */ - { 272, -3 }, /* (226) grouplist ::= grouplist COMMA item */ - { 272, -1 }, /* (227) grouplist ::= item */ - { 255, 0 }, /* (228) having_opt ::= */ - { 255, -2 }, /* (229) having_opt ::= HAVING expr */ - { 258, 0 }, /* (230) limit_opt ::= */ - { 258, -2 }, /* (231) limit_opt ::= LIMIT signed */ - { 258, -4 }, /* (232) limit_opt ::= LIMIT signed OFFSET signed */ - { 258, -4 }, /* (233) limit_opt ::= LIMIT signed COMMA signed */ - { 257, 0 }, /* (234) slimit_opt ::= */ - { 257, -2 }, /* (235) slimit_opt ::= SLIMIT signed */ - { 257, -4 }, /* (236) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 257, -4 }, /* (237) slimit_opt ::= SLIMIT signed COMMA signed */ - { 248, 0 }, /* (238) where_opt ::= */ - { 248, -2 }, /* (239) where_opt ::= WHERE expr */ - { 262, -3 }, /* (240) expr ::= LP expr RP */ - { 262, -1 }, /* (241) expr ::= ID */ - { 262, -3 }, /* (242) expr ::= ID DOT ID */ - { 262, -3 }, /* (243) expr ::= ID DOT STAR */ - { 262, -1 }, /* (244) expr ::= INTEGER */ - { 262, -2 }, /* (245) expr ::= MINUS INTEGER */ - { 262, -2 }, /* (246) expr ::= PLUS INTEGER */ - { 262, -1 }, /* (247) expr ::= FLOAT */ - { 262, -2 }, /* (248) expr ::= MINUS FLOAT */ - { 262, -2 }, /* (249) expr ::= PLUS FLOAT */ - { 262, -1 }, /* (250) expr ::= STRING */ - { 262, -1 }, /* (251) expr ::= NOW */ - { 262, -1 }, /* (252) expr ::= VARIABLE */ - { 262, -2 }, /* (253) expr ::= PLUS VARIABLE */ - { 262, -2 }, /* (254) expr ::= MINUS VARIABLE */ - { 262, -1 }, /* (255) expr ::= BOOL */ - { 262, -1 }, /* (256) expr ::= NULL */ - { 262, -4 }, /* (257) expr ::= ID LP exprlist RP */ - { 262, -4 }, /* (258) expr ::= ID LP STAR RP */ - { 262, -3 }, /* (259) expr ::= expr IS NULL */ - { 262, -4 }, /* (260) expr ::= expr IS NOT NULL */ - { 262, -3 }, /* (261) expr ::= expr LT expr */ - { 262, -3 }, /* (262) expr ::= expr GT expr */ - { 262, -3 }, /* (263) expr ::= expr LE expr */ - { 262, -3 }, /* (264) expr ::= expr GE expr */ - { 262, -3 }, /* (265) expr ::= expr NE expr */ - { 262, -3 }, /* (266) expr ::= expr EQ expr */ - { 262, -5 }, /* (267) expr ::= expr BETWEEN expr AND expr */ - { 262, -3 }, /* (268) expr ::= expr AND expr */ - { 262, -3 }, /* (269) expr ::= expr OR expr */ - { 262, -3 }, /* (270) expr ::= expr PLUS expr */ - { 262, -3 }, /* (271) expr ::= expr MINUS expr */ - { 262, -3 }, /* (272) expr ::= expr STAR expr */ - { 262, -3 }, /* (273) expr ::= expr SLASH expr */ - { 262, -3 }, /* (274) expr ::= expr REM expr */ - { 262, -3 }, /* (275) expr ::= expr LIKE expr */ - { 262, -3 }, /* (276) expr ::= expr MATCH expr */ - { 262, -3 }, /* (277) expr ::= expr NMATCH expr */ - { 262, -5 }, /* (278) expr ::= expr IN LP exprlist RP */ - { 200, -3 }, /* (279) exprlist ::= exprlist COMMA expritem */ - { 200, -1 }, /* (280) exprlist ::= expritem */ - { 273, -1 }, /* (281) expritem ::= expr */ - { 273, 0 }, /* (282) expritem ::= */ - { 193, -3 }, /* (283) cmd ::= RESET QUERY CACHE */ - { 193, -3 }, /* (284) cmd ::= SYNCDB ids REPLICA */ - { 193, -7 }, /* (285) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 193, -7 }, /* (286) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 193, -7 }, /* (287) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ - { 193, -7 }, /* (288) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 193, -7 }, /* (289) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 193, -8 }, /* (290) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 193, -9 }, /* (291) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 193, -7 }, /* (292) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ - { 193, -7 }, /* (293) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - { 193, -7 }, /* (294) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - { 193, -7 }, /* (295) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ - { 193, -7 }, /* (296) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - { 193, -7 }, /* (297) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - { 193, -8 }, /* (298) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - { 193, -9 }, /* (299) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ - { 193, -7 }, /* (300) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ - { 193, -3 }, /* (301) cmd ::= KILL CONNECTION INTEGER */ - { 193, -5 }, /* (302) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 193, -5 }, /* (303) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 193, -1 }, /* (0) program ::= cmd */ + { 194, -2 }, /* (1) cmd ::= SHOW DATABASES */ + { 194, -2 }, /* (2) cmd ::= SHOW TOPICS */ + { 194, -2 }, /* (3) cmd ::= SHOW FUNCTIONS */ + { 194, -2 }, /* (4) cmd ::= SHOW MNODES */ + { 194, -2 }, /* (5) cmd ::= SHOW DNODES */ + { 194, -2 }, /* (6) cmd ::= SHOW ACCOUNTS */ + { 194, -2 }, /* (7) cmd ::= SHOW USERS */ + { 194, -2 }, /* (8) cmd ::= SHOW MODULES */ + { 194, -2 }, /* (9) cmd ::= SHOW QUERIES */ + { 194, -2 }, /* (10) cmd ::= SHOW CONNECTIONS */ + { 194, -2 }, /* (11) cmd ::= SHOW STREAMS */ + { 194, -2 }, /* (12) cmd ::= SHOW VARIABLES */ + { 194, -2 }, /* (13) cmd ::= SHOW SCORES */ + { 194, -2 }, /* (14) cmd ::= SHOW GRANTS */ + { 194, -2 }, /* (15) cmd ::= SHOW VNODES */ + { 194, -3 }, /* (16) cmd ::= SHOW VNODES ids */ + { 196, 0 }, /* (17) dbPrefix ::= */ + { 196, -2 }, /* (18) dbPrefix ::= ids DOT */ + { 197, 0 }, /* (19) cpxName ::= */ + { 197, -2 }, /* (20) cpxName ::= DOT ids */ + { 194, -5 }, /* (21) cmd ::= SHOW CREATE TABLE ids cpxName */ + { 194, -5 }, /* (22) cmd ::= SHOW CREATE STABLE ids cpxName */ + { 194, -4 }, /* (23) cmd ::= SHOW CREATE DATABASE ids */ + { 194, -3 }, /* (24) cmd ::= SHOW dbPrefix TABLES */ + { 194, -5 }, /* (25) cmd ::= SHOW dbPrefix TABLES LIKE ids */ + { 194, -3 }, /* (26) cmd ::= SHOW dbPrefix STABLES */ + { 194, -5 }, /* (27) cmd ::= SHOW dbPrefix STABLES LIKE ids */ + { 194, -3 }, /* (28) cmd ::= SHOW dbPrefix VGROUPS */ + { 194, -4 }, /* (29) cmd ::= SHOW dbPrefix VGROUPS ids */ + { 194, -5 }, /* (30) cmd ::= DROP TABLE ifexists ids cpxName */ + { 194, -5 }, /* (31) cmd ::= DROP STABLE ifexists ids cpxName */ + { 194, -4 }, /* (32) cmd ::= DROP DATABASE ifexists ids */ + { 194, -4 }, /* (33) cmd ::= DROP TOPIC ifexists ids */ + { 194, -3 }, /* (34) cmd ::= DROP FUNCTION ids */ + { 194, -3 }, /* (35) cmd ::= DROP DNODE ids */ + { 194, -3 }, /* (36) cmd ::= DROP USER ids */ + { 194, -3 }, /* (37) cmd ::= DROP ACCOUNT ids */ + { 194, -2 }, /* (38) cmd ::= USE ids */ + { 194, -3 }, /* (39) cmd ::= DESCRIBE ids cpxName */ + { 194, -3 }, /* (40) cmd ::= DESC ids cpxName */ + { 194, -5 }, /* (41) cmd ::= ALTER USER ids PASS ids */ + { 194, -5 }, /* (42) cmd ::= ALTER USER ids PRIVILEGE ids */ + { 194, -4 }, /* (43) cmd ::= ALTER DNODE ids ids */ + { 194, -5 }, /* (44) cmd ::= ALTER DNODE ids ids ids */ + { 194, -3 }, /* (45) cmd ::= ALTER LOCAL ids */ + { 194, -4 }, /* (46) cmd ::= ALTER LOCAL ids ids */ + { 194, -4 }, /* (47) cmd ::= ALTER DATABASE ids alter_db_optr */ + { 194, -4 }, /* (48) cmd ::= ALTER ACCOUNT ids acct_optr */ + { 194, -6 }, /* (49) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ + { 194, -6 }, /* (50) cmd ::= COMPACT VNODES IN LP exprlist RP */ + { 195, -1 }, /* (51) ids ::= ID */ + { 198, -2 }, /* (52) ifexists ::= IF EXISTS */ + { 198, 0 }, /* (53) ifexists ::= */ + { 202, -3 }, /* (54) ifnotexists ::= IF NOT EXISTS */ + { 202, 0 }, /* (55) ifnotexists ::= */ + { 194, -5 }, /* (56) cmd ::= CREATE DNODE ids PORT ids */ + { 194, -5 }, /* (57) cmd ::= CREATE DNODE IPTOKEN PORT ids */ + { 194, -6 }, /* (58) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ + { 194, -5 }, /* (59) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ + { 194, -8 }, /* (60) cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + { 194, -9 }, /* (61) cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ + { 194, -5 }, /* (62) cmd ::= CREATE USER ids PASS ids */ + { 205, 0 }, /* (63) bufsize ::= */ + { 205, -2 }, /* (64) bufsize ::= BUFSIZE INTEGER */ + { 206, 0 }, /* (65) pps ::= */ + { 206, -2 }, /* (66) pps ::= PPS INTEGER */ + { 207, 0 }, /* (67) tseries ::= */ + { 207, -2 }, /* (68) tseries ::= TSERIES INTEGER */ + { 208, 0 }, /* (69) dbs ::= */ + { 208, -2 }, /* (70) dbs ::= DBS INTEGER */ + { 209, 0 }, /* (71) streams ::= */ + { 209, -2 }, /* (72) streams ::= STREAMS INTEGER */ + { 210, 0 }, /* (73) storage ::= */ + { 210, -2 }, /* (74) storage ::= STORAGE INTEGER */ + { 211, 0 }, /* (75) qtime ::= */ + { 211, -2 }, /* (76) qtime ::= QTIME INTEGER */ + { 212, 0 }, /* (77) users ::= */ + { 212, -2 }, /* (78) users ::= USERS INTEGER */ + { 213, 0 }, /* (79) conns ::= */ + { 213, -2 }, /* (80) conns ::= CONNS INTEGER */ + { 214, 0 }, /* (81) state ::= */ + { 214, -2 }, /* (82) state ::= STATE ids */ + { 200, -9 }, /* (83) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ + { 215, -3 }, /* (84) intitemlist ::= intitemlist COMMA intitem */ + { 215, -1 }, /* (85) intitemlist ::= intitem */ + { 216, -1 }, /* (86) intitem ::= INTEGER */ + { 217, -2 }, /* (87) keep ::= KEEP intitemlist */ + { 218, -2 }, /* (88) cache ::= CACHE INTEGER */ + { 219, -2 }, /* (89) replica ::= REPLICA INTEGER */ + { 220, -2 }, /* (90) quorum ::= QUORUM INTEGER */ + { 221, -2 }, /* (91) days ::= DAYS INTEGER */ + { 222, -2 }, /* (92) minrows ::= MINROWS INTEGER */ + { 223, -2 }, /* (93) maxrows ::= MAXROWS INTEGER */ + { 224, -2 }, /* (94) blocks ::= BLOCKS INTEGER */ + { 225, -2 }, /* (95) ctime ::= CTIME INTEGER */ + { 226, -2 }, /* (96) wal ::= WAL INTEGER */ + { 227, -2 }, /* (97) fsync ::= FSYNC INTEGER */ + { 228, -2 }, /* (98) comp ::= COMP INTEGER */ + { 229, -2 }, /* (99) prec ::= PRECISION STRING */ + { 230, -2 }, /* (100) update ::= UPDATE INTEGER */ + { 231, -2 }, /* (101) cachelast ::= CACHELAST INTEGER */ + { 232, -2 }, /* (102) vgroups ::= VGROUPS INTEGER */ + { 233, -3 }, /* (103) stream_mode ::= STREAM MODE INTEGER */ + { 203, 0 }, /* (104) db_optr ::= */ + { 203, -2 }, /* (105) db_optr ::= db_optr cache */ + { 203, -2 }, /* (106) db_optr ::= db_optr replica */ + { 203, -2 }, /* (107) db_optr ::= db_optr quorum */ + { 203, -2 }, /* (108) db_optr ::= db_optr days */ + { 203, -2 }, /* (109) db_optr ::= db_optr minrows */ + { 203, -2 }, /* (110) db_optr ::= db_optr maxrows */ + { 203, -2 }, /* (111) db_optr ::= db_optr blocks */ + { 203, -2 }, /* (112) db_optr ::= db_optr ctime */ + { 203, -2 }, /* (113) db_optr ::= db_optr wal */ + { 203, -2 }, /* (114) db_optr ::= db_optr fsync */ + { 203, -2 }, /* (115) db_optr ::= db_optr comp */ + { 203, -2 }, /* (116) db_optr ::= db_optr prec */ + { 203, -2 }, /* (117) db_optr ::= db_optr keep */ + { 203, -2 }, /* (118) db_optr ::= db_optr update */ + { 203, -2 }, /* (119) db_optr ::= db_optr cachelast */ + { 203, -2 }, /* (120) db_optr ::= db_optr vgroups */ + { 203, -2 }, /* (121) db_optr ::= db_optr stream_mode */ + { 199, 0 }, /* (122) alter_db_optr ::= */ + { 199, -2 }, /* (123) alter_db_optr ::= alter_db_optr replica */ + { 199, -2 }, /* (124) alter_db_optr ::= alter_db_optr quorum */ + { 199, -2 }, /* (125) alter_db_optr ::= alter_db_optr keep */ + { 199, -2 }, /* (126) alter_db_optr ::= alter_db_optr blocks */ + { 199, -2 }, /* (127) alter_db_optr ::= alter_db_optr comp */ + { 199, -2 }, /* (128) alter_db_optr ::= alter_db_optr update */ + { 199, -2 }, /* (129) alter_db_optr ::= alter_db_optr cachelast */ + { 204, -1 }, /* (130) typename ::= ids */ + { 204, -4 }, /* (131) typename ::= ids LP signed RP */ + { 204, -2 }, /* (132) typename ::= ids UNSIGNED */ + { 234, -1 }, /* (133) signed ::= INTEGER */ + { 234, -2 }, /* (134) signed ::= PLUS INTEGER */ + { 234, -2 }, /* (135) signed ::= MINUS INTEGER */ + { 194, -3 }, /* (136) cmd ::= CREATE TABLE create_table_args */ + { 194, -3 }, /* (137) cmd ::= CREATE TABLE create_stable_args */ + { 194, -3 }, /* (138) cmd ::= CREATE STABLE create_stable_args */ + { 194, -3 }, /* (139) cmd ::= CREATE TABLE create_table_list */ + { 237, -1 }, /* (140) create_table_list ::= create_from_stable */ + { 237, -2 }, /* (141) create_table_list ::= create_table_list create_from_stable */ + { 235, -6 }, /* (142) create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ + { 236, -10 }, /* (143) create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ + { 238, -10 }, /* (144) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist1 RP */ + { 238, -13 }, /* (145) create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist1 RP */ + { 241, -3 }, /* (146) tagNamelist ::= tagNamelist COMMA ids */ + { 241, -1 }, /* (147) tagNamelist ::= ids */ + { 235, -5 }, /* (148) create_table_args ::= ifnotexists ids cpxName AS select */ + { 239, -3 }, /* (149) columnlist ::= columnlist COMMA column */ + { 239, -1 }, /* (150) columnlist ::= column */ + { 243, -2 }, /* (151) column ::= ids typename */ + { 240, -3 }, /* (152) tagitemlist1 ::= tagitemlist1 COMMA tagitem1 */ + { 240, -1 }, /* (153) tagitemlist1 ::= tagitem1 */ + { 244, -2 }, /* (154) tagitem1 ::= MINUS INTEGER */ + { 244, -2 }, /* (155) tagitem1 ::= MINUS FLOAT */ + { 244, -2 }, /* (156) tagitem1 ::= PLUS INTEGER */ + { 244, -2 }, /* (157) tagitem1 ::= PLUS FLOAT */ + { 244, -1 }, /* (158) tagitem1 ::= INTEGER */ + { 244, -1 }, /* (159) tagitem1 ::= FLOAT */ + { 244, -1 }, /* (160) tagitem1 ::= STRING */ + { 244, -1 }, /* (161) tagitem1 ::= BOOL */ + { 244, -1 }, /* (162) tagitem1 ::= NULL */ + { 244, -1 }, /* (163) tagitem1 ::= NOW */ + { 245, -3 }, /* (164) tagitemlist ::= tagitemlist COMMA tagitem */ + { 245, -1 }, /* (165) tagitemlist ::= tagitem */ + { 246, -1 }, /* (166) tagitem ::= INTEGER */ + { 246, -1 }, /* (167) tagitem ::= FLOAT */ + { 246, -1 }, /* (168) tagitem ::= STRING */ + { 246, -1 }, /* (169) tagitem ::= BOOL */ + { 246, -1 }, /* (170) tagitem ::= NULL */ + { 246, -1 }, /* (171) tagitem ::= NOW */ + { 246, -2 }, /* (172) tagitem ::= MINUS INTEGER */ + { 246, -2 }, /* (173) tagitem ::= MINUS FLOAT */ + { 246, -2 }, /* (174) tagitem ::= PLUS INTEGER */ + { 246, -2 }, /* (175) tagitem ::= PLUS FLOAT */ + { 242, -14 }, /* (176) select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ + { 242, -3 }, /* (177) select ::= LP select RP */ + { 260, -1 }, /* (178) union ::= select */ + { 260, -4 }, /* (179) union ::= union UNION ALL select */ + { 260, -3 }, /* (180) union ::= union UNION select */ + { 194, -1 }, /* (181) cmd ::= union */ + { 242, -2 }, /* (182) select ::= SELECT selcollist */ + { 261, -2 }, /* (183) sclp ::= selcollist COMMA */ + { 261, 0 }, /* (184) sclp ::= */ + { 247, -4 }, /* (185) selcollist ::= sclp distinct expr as */ + { 247, -2 }, /* (186) selcollist ::= sclp STAR */ + { 264, -2 }, /* (187) as ::= AS ids */ + { 264, -1 }, /* (188) as ::= ids */ + { 264, 0 }, /* (189) as ::= */ + { 262, -1 }, /* (190) distinct ::= DISTINCT */ + { 262, 0 }, /* (191) distinct ::= */ + { 248, -2 }, /* (192) from ::= FROM tablelist */ + { 248, -2 }, /* (193) from ::= FROM sub */ + { 266, -3 }, /* (194) sub ::= LP union RP */ + { 266, -4 }, /* (195) sub ::= LP union RP ids */ + { 266, -6 }, /* (196) sub ::= sub COMMA LP union RP ids */ + { 265, -2 }, /* (197) tablelist ::= ids cpxName */ + { 265, -3 }, /* (198) tablelist ::= ids cpxName ids */ + { 265, -4 }, /* (199) tablelist ::= tablelist COMMA ids cpxName */ + { 265, -5 }, /* (200) tablelist ::= tablelist COMMA ids cpxName ids */ + { 267, -1 }, /* (201) tmvar ::= VARIABLE */ + { 250, -4 }, /* (202) interval_option ::= intervalKey LP tmvar RP */ + { 250, -6 }, /* (203) interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ + { 250, 0 }, /* (204) interval_option ::= */ + { 268, -1 }, /* (205) intervalKey ::= INTERVAL */ + { 268, -1 }, /* (206) intervalKey ::= EVERY */ + { 252, 0 }, /* (207) session_option ::= */ + { 252, -7 }, /* (208) session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ + { 253, 0 }, /* (209) windowstate_option ::= */ + { 253, -4 }, /* (210) windowstate_option ::= STATE_WINDOW LP ids RP */ + { 254, 0 }, /* (211) fill_opt ::= */ + { 254, -6 }, /* (212) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 254, -4 }, /* (213) fill_opt ::= FILL LP ID RP */ + { 251, -4 }, /* (214) sliding_opt ::= SLIDING LP tmvar RP */ + { 251, 0 }, /* (215) sliding_opt ::= */ + { 257, 0 }, /* (216) orderby_opt ::= */ + { 257, -3 }, /* (217) orderby_opt ::= ORDER BY sortlist */ + { 269, -4 }, /* (218) sortlist ::= sortlist COMMA item sortorder */ + { 269, -2 }, /* (219) sortlist ::= item sortorder */ + { 271, -2 }, /* (220) item ::= ids cpxName */ + { 272, -1 }, /* (221) sortorder ::= ASC */ + { 272, -1 }, /* (222) sortorder ::= DESC */ + { 272, 0 }, /* (223) sortorder ::= */ + { 255, 0 }, /* (224) groupby_opt ::= */ + { 255, -3 }, /* (225) groupby_opt ::= GROUP BY grouplist */ + { 273, -3 }, /* (226) grouplist ::= grouplist COMMA item */ + { 273, -1 }, /* (227) grouplist ::= item */ + { 256, 0 }, /* (228) having_opt ::= */ + { 256, -2 }, /* (229) having_opt ::= HAVING expr */ + { 259, 0 }, /* (230) limit_opt ::= */ + { 259, -2 }, /* (231) limit_opt ::= LIMIT signed */ + { 259, -4 }, /* (232) limit_opt ::= LIMIT signed OFFSET signed */ + { 259, -4 }, /* (233) limit_opt ::= LIMIT signed COMMA signed */ + { 258, 0 }, /* (234) slimit_opt ::= */ + { 258, -2 }, /* (235) slimit_opt ::= SLIMIT signed */ + { 258, -4 }, /* (236) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 258, -4 }, /* (237) slimit_opt ::= SLIMIT signed COMMA signed */ + { 249, 0 }, /* (238) where_opt ::= */ + { 249, -2 }, /* (239) where_opt ::= WHERE expr */ + { 263, -3 }, /* (240) expr ::= LP expr RP */ + { 263, -1 }, /* (241) expr ::= ID */ + { 263, -3 }, /* (242) expr ::= ID DOT ID */ + { 263, -3 }, /* (243) expr ::= ID DOT STAR */ + { 263, -1 }, /* (244) expr ::= INTEGER */ + { 263, -2 }, /* (245) expr ::= MINUS INTEGER */ + { 263, -2 }, /* (246) expr ::= PLUS INTEGER */ + { 263, -1 }, /* (247) expr ::= FLOAT */ + { 263, -2 }, /* (248) expr ::= MINUS FLOAT */ + { 263, -2 }, /* (249) expr ::= PLUS FLOAT */ + { 263, -1 }, /* (250) expr ::= STRING */ + { 263, -1 }, /* (251) expr ::= NOW */ + { 263, -1 }, /* (252) expr ::= VARIABLE */ + { 263, -2 }, /* (253) expr ::= PLUS VARIABLE */ + { 263, -2 }, /* (254) expr ::= MINUS VARIABLE */ + { 263, -1 }, /* (255) expr ::= BOOL */ + { 263, -1 }, /* (256) expr ::= NULL */ + { 263, -4 }, /* (257) expr ::= ID LP exprlist RP */ + { 263, -4 }, /* (258) expr ::= ID LP STAR RP */ + { 263, -3 }, /* (259) expr ::= expr IS NULL */ + { 263, -4 }, /* (260) expr ::= expr IS NOT NULL */ + { 263, -3 }, /* (261) expr ::= expr LT expr */ + { 263, -3 }, /* (262) expr ::= expr GT expr */ + { 263, -3 }, /* (263) expr ::= expr LE expr */ + { 263, -3 }, /* (264) expr ::= expr GE expr */ + { 263, -3 }, /* (265) expr ::= expr NE expr */ + { 263, -3 }, /* (266) expr ::= expr EQ expr */ + { 263, -5 }, /* (267) expr ::= expr BETWEEN expr AND expr */ + { 263, -3 }, /* (268) expr ::= expr AND expr */ + { 263, -3 }, /* (269) expr ::= expr OR expr */ + { 263, -3 }, /* (270) expr ::= expr PLUS expr */ + { 263, -3 }, /* (271) expr ::= expr MINUS expr */ + { 263, -3 }, /* (272) expr ::= expr STAR expr */ + { 263, -3 }, /* (273) expr ::= expr SLASH expr */ + { 263, -3 }, /* (274) expr ::= expr REM expr */ + { 263, -3 }, /* (275) expr ::= expr LIKE expr */ + { 263, -3 }, /* (276) expr ::= expr MATCH expr */ + { 263, -3 }, /* (277) expr ::= expr NMATCH expr */ + { 263, -5 }, /* (278) expr ::= expr IN LP exprlist RP */ + { 201, -3 }, /* (279) exprlist ::= exprlist COMMA expritem */ + { 201, -1 }, /* (280) exprlist ::= expritem */ + { 274, -1 }, /* (281) expritem ::= expr */ + { 274, 0 }, /* (282) expritem ::= */ + { 194, -3 }, /* (283) cmd ::= RESET QUERY CACHE */ + { 194, -3 }, /* (284) cmd ::= SYNCDB ids REPLICA */ + { 194, -7 }, /* (285) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 194, -7 }, /* (286) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 194, -7 }, /* (287) cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ + { 194, -7 }, /* (288) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 194, -7 }, /* (289) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 194, -8 }, /* (290) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 194, -9 }, /* (291) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 194, -7 }, /* (292) cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ + { 194, -7 }, /* (293) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + { 194, -7 }, /* (294) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + { 194, -7 }, /* (295) cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ + { 194, -7 }, /* (296) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + { 194, -7 }, /* (297) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + { 194, -8 }, /* (298) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + { 194, -9 }, /* (299) cmd ::= ALTER STABLE ids cpxName SET TAG ids EQ tagitem */ + { 194, -7 }, /* (300) cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ + { 194, -3 }, /* (301) cmd ::= KILL CONNECTION INTEGER */ + { 194, -5 }, /* (302) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 194, -5 }, /* (303) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2181,18 +2163,17 @@ static void yy_accept(yyParser*); /* Forward Declaration */ ** only called from one place, optimizing compilers will in-line it, which ** means that the extra parameters have no performance impact. */ -static YYACTIONTYPE yy_reduce( +static void yy_reduce( yyParser *yypParser, /* The parser */ unsigned int yyruleno, /* Number of the rule by which to reduce */ int yyLookahead, /* Lookahead token, or YYNOCODE if none */ ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - ParseCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - YYACTIONTYPE yyact; /* The next action */ + int yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH + ParseARG_FETCH; (void)yyLookahead; (void)yyLookaheadToken; yymsp = yypParser->yytos; @@ -2223,19 +2204,13 @@ static YYACTIONTYPE yy_reduce( #if YYSTACKDEPTH>0 if( yypParser->yytos>=yypParser->yystackEnd ){ yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; + return; } #else if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ if( yyGrowStack(yypParser) ){ yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; + return; } yymsp = yypParser->yytos; } @@ -2432,16 +2407,16 @@ static YYACTIONTYPE yy_reduce( { setDCLSqlElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 47: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy326, &t);} +{ SToken t = {0}; setCreateDbInfo(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy42, &t);} break; case 48: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy523);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy341);} break; case 49: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy523);} +{ setCreateAcctSql(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy341);} break; case 50: /* cmd ::= COMPACT VNODES IN LP exprlist RP */ -{ setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, yymsp[-1].minor.yy225);} +{ setCompactVnodeSql(pInfo, TSDB_SQL_COMPACT_VNODE, yymsp[-1].minor.yy131);} break; case 51: /* ids ::= ID */ {yylhsminor.yy0 = yymsp[0].minor.yy0; } @@ -2463,16 +2438,16 @@ static YYACTIONTYPE yy_reduce( { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} break; case 58: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy523);} +{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy341);} break; case 59: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy326, &yymsp[-2].minor.yy0);} +{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy42, &yymsp[-2].minor.yy0);} break; case 60: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy16, &yymsp[0].minor.yy0, 1);} +{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy290, &yymsp[0].minor.yy0, 1);} break; case 61: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */ -{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy16, &yymsp[0].minor.yy0, 2);} +{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy290, &yymsp[0].minor.yy0, 2);} break; case 62: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2503,38 +2478,38 @@ static YYACTIONTYPE yy_reduce( break; case 83: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy523.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy523.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy523.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy523.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy523.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy523.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy523.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy523.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy523.stat = yymsp[0].minor.yy0; + yylhsminor.yy341.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yylhsminor.yy341.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yylhsminor.yy341.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yylhsminor.yy341.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yylhsminor.yy341.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yylhsminor.yy341.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy341.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yylhsminor.yy341.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yylhsminor.yy341.stat = yymsp[0].minor.yy0; } - yymsp[-8].minor.yy523 = yylhsminor.yy523; + yymsp[-8].minor.yy341 = yylhsminor.yy341; break; case 84: /* intitemlist ::= intitemlist COMMA intitem */ case 164: /* tagitemlist ::= tagitemlist COMMA tagitem */ yytestcase(yyruleno==164); -{ yylhsminor.yy225 = tListItemAppend(yymsp[-2].minor.yy225, &yymsp[0].minor.yy1, -1); } - yymsp[-2].minor.yy225 = yylhsminor.yy225; +{ yylhsminor.yy131 = tListItemAppend(yymsp[-2].minor.yy131, &yymsp[0].minor.yy43, -1); } + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 85: /* intitemlist ::= intitem */ case 165: /* tagitemlist ::= tagitem */ yytestcase(yyruleno==165); -{ yylhsminor.yy225 = tListItemAppend(NULL, &yymsp[0].minor.yy1, -1); } - yymsp[0].minor.yy225 = yylhsminor.yy225; +{ yylhsminor.yy131 = tListItemAppend(NULL, &yymsp[0].minor.yy43, -1); } + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 86: /* intitem ::= INTEGER */ case 166: /* tagitem ::= INTEGER */ yytestcase(yyruleno==166); case 167: /* tagitem ::= FLOAT */ yytestcase(yyruleno==167); case 168: /* tagitem ::= STRING */ yytestcase(yyruleno==168); case 169: /* tagitem ::= BOOL */ yytestcase(yyruleno==169); -{ toTSDBType(yymsp[0].minor.yy0.type); taosVariantCreate(&yylhsminor.yy1, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type); } - yymsp[0].minor.yy1 = yylhsminor.yy1; +{ toTSDBType(yymsp[0].minor.yy0.type); taosVariantCreate(&yylhsminor.yy43, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type); } + yymsp[0].minor.yy43 = yylhsminor.yy43; break; case 87: /* keep ::= KEEP intitemlist */ -{ yymsp[-1].minor.yy225 = yymsp[0].minor.yy225; } +{ yymsp[-1].minor.yy131 = yymsp[0].minor.yy131; } break; case 88: /* cache ::= CACHE INTEGER */ case 89: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==89); @@ -2557,225 +2532,225 @@ static YYACTIONTYPE yy_reduce( { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; } break; case 104: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy326);} +{setDefaultCreateDbOption(&yymsp[1].minor.yy42);} break; case 105: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 106: /* db_optr ::= db_optr replica */ case 123: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==123); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 107: /* db_optr ::= db_optr quorum */ case 124: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==124); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 108: /* db_optr ::= db_optr days */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 109: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 110: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 111: /* db_optr ::= db_optr blocks */ case 126: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==126); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 112: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 113: /* db_optr ::= db_optr wal */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 114: /* db_optr ::= db_optr fsync */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 115: /* db_optr ::= db_optr comp */ case 127: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==127); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 116: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.precision = yymsp[0].minor.yy0; } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 117: /* db_optr ::= db_optr keep */ case 125: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==125); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.keep = yymsp[0].minor.yy225; } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.keep = yymsp[0].minor.yy131; } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 118: /* db_optr ::= db_optr update */ case 128: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==128); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 119: /* db_optr ::= db_optr cachelast */ case 129: /* alter_db_optr ::= alter_db_optr cachelast */ yytestcase(yyruleno==129); -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.cachelast = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 120: /* db_optr ::= db_optr vgroups */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.numOfVgroups = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.numOfVgroups = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 121: /* db_optr ::= db_optr stream_mode */ -{ yylhsminor.yy326 = yymsp[-1].minor.yy326; yylhsminor.yy326.streamMode = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy326 = yylhsminor.yy326; +{ yylhsminor.yy42 = yymsp[-1].minor.yy42; yylhsminor.yy42.streamMode = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[-1].minor.yy42 = yylhsminor.yy42; break; case 122: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy326);} +{ setDefaultCreateDbOption(&yymsp[1].minor.yy42);} break; case 130: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSetColumnType (&yylhsminor.yy16, &yymsp[0].minor.yy0); + tSetColumnType (&yylhsminor.yy290, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy16 = yylhsminor.yy16; + yymsp[0].minor.yy290 = yylhsminor.yy290; break; case 131: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy61 <= 0) { + if (yymsp[-1].minor.yy459 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSetColumnType(&yylhsminor.yy16, &yymsp[-3].minor.yy0); + tSetColumnType(&yylhsminor.yy290, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy61; // negative value of name length - tSetColumnType(&yylhsminor.yy16, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy459; // negative value of name length + tSetColumnType(&yylhsminor.yy290, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy16 = yylhsminor.yy16; + yymsp[-3].minor.yy290 = yylhsminor.yy290; break; case 132: /* typename ::= ids UNSIGNED */ { yymsp[-1].minor.yy0.type = 0; yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z); - tSetColumnType (&yylhsminor.yy16, &yymsp[-1].minor.yy0); + tSetColumnType (&yylhsminor.yy290, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy16 = yylhsminor.yy16; + yymsp[-1].minor.yy290 = yylhsminor.yy290; break; case 133: /* signed ::= INTEGER */ -{ yylhsminor.yy61 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy61 = yylhsminor.yy61; +{ yylhsminor.yy459 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + yymsp[0].minor.yy459 = yylhsminor.yy459; break; case 134: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy61 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy459 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; case 135: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy61 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} +{ yymsp[-1].minor.yy459 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; case 139: /* cmd ::= CREATE TABLE create_table_list */ -{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy482;} +{ pInfo->type = TSDB_SQL_CREATE_TABLE; pInfo->pCreateTableInfo = yymsp[0].minor.yy272;} break; case 140: /* create_table_list ::= create_from_stable */ { SCreateTableSql* pCreateTable = calloc(1, sizeof(SCreateTableSql)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); - taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy184); + taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy96); pCreateTable->type = TSDB_SQL_CREATE_TABLE; - yylhsminor.yy482 = pCreateTable; + yylhsminor.yy272 = pCreateTable; } - yymsp[0].minor.yy482 = yylhsminor.yy482; + yymsp[0].minor.yy272 = yylhsminor.yy272; break; case 141: /* create_table_list ::= create_table_list create_from_stable */ { - taosArrayPush(yymsp[-1].minor.yy482->childTableInfo, &yymsp[0].minor.yy184); - yylhsminor.yy482 = yymsp[-1].minor.yy482; + taosArrayPush(yymsp[-1].minor.yy272->childTableInfo, &yymsp[0].minor.yy96); + yylhsminor.yy272 = yymsp[-1].minor.yy272; } - yymsp[-1].minor.yy482 = yylhsminor.yy482; + yymsp[-1].minor.yy272 = yylhsminor.yy272; break; case 142: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ { - yylhsminor.yy482 = tSetCreateTableInfo(yymsp[-1].minor.yy225, NULL, NULL, TSDB_SQL_CREATE_TABLE); - setSqlInfo(pInfo, yylhsminor.yy482, NULL, TSDB_SQL_CREATE_TABLE); + yylhsminor.yy272 = tSetCreateTableInfo(yymsp[-1].minor.yy131, NULL, NULL, TSDB_SQL_CREATE_TABLE); + setSqlInfo(pInfo, yylhsminor.yy272, NULL, TSDB_SQL_CREATE_TABLE); yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-4].minor.yy0, &yymsp[-5].minor.yy0); } - yymsp[-5].minor.yy482 = yylhsminor.yy482; + yymsp[-5].minor.yy272 = yylhsminor.yy272; break; case 143: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ { - yylhsminor.yy482 = tSetCreateTableInfo(yymsp[-5].minor.yy225, yymsp[-1].minor.yy225, NULL, TSDB_SQL_CREATE_STABLE); - setSqlInfo(pInfo, yylhsminor.yy482, NULL, TSDB_SQL_CREATE_STABLE); + yylhsminor.yy272 = tSetCreateTableInfo(yymsp[-5].minor.yy131, yymsp[-1].minor.yy131, NULL, TSDB_SQL_CREATE_STABLE); + setSqlInfo(pInfo, yylhsminor.yy272, NULL, TSDB_SQL_CREATE_STABLE); yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy482 = yylhsminor.yy482; + yymsp[-9].minor.yy272 = yylhsminor.yy272; break; case 144: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName TAGS LP tagitemlist1 RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; - yylhsminor.yy184 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy225, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); + yylhsminor.yy96 = createNewChildTableInfo(&yymsp[-5].minor.yy0, NULL, yymsp[-1].minor.yy131, &yymsp[-8].minor.yy0, &yymsp[-9].minor.yy0); } - yymsp[-9].minor.yy184 = yylhsminor.yy184; + yymsp[-9].minor.yy96 = yylhsminor.yy96; break; case 145: /* create_from_stable ::= ifnotexists ids cpxName USING ids cpxName LP tagNamelist RP TAGS LP tagitemlist1 RP */ { yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; yymsp[-11].minor.yy0.n += yymsp[-10].minor.yy0.n; - yylhsminor.yy184 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy225, yymsp[-1].minor.yy225, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); + yylhsminor.yy96 = createNewChildTableInfo(&yymsp[-8].minor.yy0, yymsp[-5].minor.yy131, yymsp[-1].minor.yy131, &yymsp[-11].minor.yy0, &yymsp[-12].minor.yy0); } - yymsp[-12].minor.yy184 = yylhsminor.yy184; + yymsp[-12].minor.yy96 = yylhsminor.yy96; break; case 146: /* tagNamelist ::= tagNamelist COMMA ids */ -{taosArrayPush(yymsp[-2].minor.yy225, &yymsp[0].minor.yy0); yylhsminor.yy225 = yymsp[-2].minor.yy225; } - yymsp[-2].minor.yy225 = yylhsminor.yy225; +{taosArrayPush(yymsp[-2].minor.yy131, &yymsp[0].minor.yy0); yylhsminor.yy131 = yymsp[-2].minor.yy131; } + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 147: /* tagNamelist ::= ids */ -{yylhsminor.yy225 = taosArrayInit(4, sizeof(SToken)); taosArrayPush(yylhsminor.yy225, &yymsp[0].minor.yy0);} - yymsp[0].minor.yy225 = yylhsminor.yy225; +{yylhsminor.yy131 = taosArrayInit(4, sizeof(SToken)); taosArrayPush(yylhsminor.yy131, &yymsp[0].minor.yy0);} + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 148: /* create_table_args ::= ifnotexists ids cpxName AS select */ { -// yylhsminor.yy482 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy185, TSQL_CREATE_STREAM); -// setSqlInfo(pInfo, yylhsminor.yy482, NULL, TSDB_SQL_CREATE_TABLE); +// yylhsminor.yy272 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy256, TSQL_CREATE_STREAM); +// setSqlInfo(pInfo, yylhsminor.yy272, NULL, TSDB_SQL_CREATE_TABLE); // // yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; // setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); } - yymsp[-4].minor.yy482 = yylhsminor.yy482; + yymsp[-4].minor.yy272 = yylhsminor.yy272; break; case 149: /* columnlist ::= columnlist COMMA column */ -{taosArrayPush(yymsp[-2].minor.yy225, &yymsp[0].minor.yy16); yylhsminor.yy225 = yymsp[-2].minor.yy225; } - yymsp[-2].minor.yy225 = yylhsminor.yy225; +{taosArrayPush(yymsp[-2].minor.yy131, &yymsp[0].minor.yy290); yylhsminor.yy131 = yymsp[-2].minor.yy131; } + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 150: /* columnlist ::= column */ -{yylhsminor.yy225 = taosArrayInit(4, sizeof(SField)); taosArrayPush(yylhsminor.yy225, &yymsp[0].minor.yy16);} - yymsp[0].minor.yy225 = yylhsminor.yy225; +{yylhsminor.yy131 = taosArrayInit(4, sizeof(SField)); taosArrayPush(yylhsminor.yy131, &yymsp[0].minor.yy290);} + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 151: /* column ::= ids typename */ { - tSetColumnInfo(&yylhsminor.yy16, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy16); + tSetColumnInfo(&yylhsminor.yy290, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy290); } - yymsp[-1].minor.yy16 = yylhsminor.yy16; + yymsp[-1].minor.yy290 = yylhsminor.yy290; break; case 152: /* tagitemlist1 ::= tagitemlist1 COMMA tagitem1 */ -{ taosArrayPush(yymsp[-2].minor.yy225, &yymsp[0].minor.yy0); yylhsminor.yy225 = yymsp[-2].minor.yy225;} - yymsp[-2].minor.yy225 = yylhsminor.yy225; +{ taosArrayPush(yymsp[-2].minor.yy131, &yymsp[0].minor.yy0); yylhsminor.yy131 = yymsp[-2].minor.yy131;} + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 153: /* tagitemlist1 ::= tagitem1 */ -{ yylhsminor.yy225 = taosArrayInit(4, sizeof(SToken)); taosArrayPush(yylhsminor.yy225, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy225 = yylhsminor.yy225; +{ yylhsminor.yy131 = taosArrayInit(4, sizeof(SToken)); taosArrayPush(yylhsminor.yy131, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 154: /* tagitem1 ::= MINUS INTEGER */ case 155: /* tagitem1 ::= MINUS FLOAT */ yytestcase(yyruleno==155); case 156: /* tagitem1 ::= PLUS INTEGER */ yytestcase(yyruleno==156); case 157: /* tagitem1 ::= PLUS FLOAT */ yytestcase(yyruleno==157); -{ yylhsminor.yy0.n = yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n; yylhsminor.yy0.type = yymsp[0].minor.yy0.type; } +{ yylhsminor.yy0.n = yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n; yylhsminor.yy0.type = yymsp[0].minor.yy0.type; yylhsminor.yy0.z = yymsp[-1].minor.yy0.z;} yymsp[-1].minor.yy0 = yylhsminor.yy0; break; case 158: /* tagitem1 ::= INTEGER */ @@ -2788,12 +2763,12 @@ static YYACTIONTYPE yy_reduce( yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 170: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; taosVariantCreate(&yylhsminor.yy1, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type); } - yymsp[0].minor.yy1 = yylhsminor.yy1; +{ yymsp[0].minor.yy0.type = 0; taosVariantCreate(&yylhsminor.yy43, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type); } + yymsp[0].minor.yy43 = yylhsminor.yy43; break; case 171: /* tagitem ::= NOW */ -{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&yylhsminor.yy1, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type);} - yymsp[0].minor.yy1 = yylhsminor.yy1; +{ yymsp[0].minor.yy0.type = TSDB_DATA_TYPE_TIMESTAMP; taosVariantCreate(&yylhsminor.yy43, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type);} + yymsp[0].minor.yy43 = yylhsminor.yy43; break; case 172: /* tagitem ::= MINUS INTEGER */ case 173: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==173); @@ -2803,60 +2778,60 @@ static YYACTIONTYPE yy_reduce( yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - taosVariantCreate(&yylhsminor.yy1, yymsp[-1].minor.yy0.z, yymsp[-1].minor.yy0.n, yymsp[-1].minor.yy0.type); + taosVariantCreate(&yylhsminor.yy43, yymsp[-1].minor.yy0.z, yymsp[-1].minor.yy0.n, yymsp[-1].minor.yy0.type); } - yymsp[-1].minor.yy1 = yylhsminor.yy1; + yymsp[-1].minor.yy43 = yylhsminor.yy43; break; case 176: /* select ::= SELECT selcollist from where_opt interval_option sliding_opt session_option windowstate_option fill_opt groupby_opt having_opt orderby_opt slimit_opt limit_opt */ { - yylhsminor.yy185 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy225, yymsp[-11].minor.yy160, yymsp[-10].minor.yy226, yymsp[-4].minor.yy225, yymsp[-2].minor.yy225, &yymsp[-9].minor.yy52, &yymsp[-7].minor.yy463, &yymsp[-6].minor.yy112, &yymsp[-8].minor.yy0, yymsp[-5].minor.yy225, &yymsp[0].minor.yy495, &yymsp[-1].minor.yy495, yymsp[-3].minor.yy226); + yylhsminor.yy256 = tSetQuerySqlNode(&yymsp[-13].minor.yy0, yymsp[-12].minor.yy131, yymsp[-11].minor.yy544, yymsp[-10].minor.yy46, yymsp[-4].minor.yy131, yymsp[-2].minor.yy131, &yymsp[-9].minor.yy530, &yymsp[-7].minor.yy39, &yymsp[-6].minor.yy538, &yymsp[-8].minor.yy0, yymsp[-5].minor.yy131, &yymsp[0].minor.yy55, &yymsp[-1].minor.yy55, yymsp[-3].minor.yy46); } - yymsp[-13].minor.yy185 = yylhsminor.yy185; + yymsp[-13].minor.yy256 = yylhsminor.yy256; break; case 177: /* select ::= LP select RP */ -{yymsp[-2].minor.yy185 = yymsp[-1].minor.yy185;} +{yymsp[-2].minor.yy256 = yymsp[-1].minor.yy256;} break; case 178: /* union ::= select */ -{ yylhsminor.yy93 = setSubclause(NULL, yymsp[0].minor.yy185); } - yymsp[0].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy303 = setSubclause(NULL, yymsp[0].minor.yy256); } + yymsp[0].minor.yy303 = yylhsminor.yy303; break; case 179: /* union ::= union UNION ALL select */ -{ yylhsminor.yy93 = appendSelectClause(yymsp[-3].minor.yy93, SQL_TYPE_UNIONALL, yymsp[0].minor.yy185); } - yymsp[-3].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy303 = appendSelectClause(yymsp[-3].minor.yy303, SQL_TYPE_UNIONALL, yymsp[0].minor.yy256); } + yymsp[-3].minor.yy303 = yylhsminor.yy303; break; case 180: /* union ::= union UNION select */ -{ yylhsminor.yy93 = appendSelectClause(yymsp[-2].minor.yy93, SQL_TYPE_UNION, yymsp[0].minor.yy185); } - yymsp[-2].minor.yy93 = yylhsminor.yy93; +{ yylhsminor.yy303 = appendSelectClause(yymsp[-2].minor.yy303, SQL_TYPE_UNION, yymsp[0].minor.yy256); } + yymsp[-2].minor.yy303 = yylhsminor.yy303; break; case 181: /* cmd ::= union */ -{ setSqlInfo(pInfo, yymsp[0].minor.yy93, NULL, TSDB_SQL_SELECT); } +{ setSqlInfo(pInfo, yymsp[0].minor.yy303, NULL, TSDB_SQL_SELECT); } break; case 182: /* select ::= SELECT selcollist */ { - yylhsminor.yy185 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy225, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yylhsminor.yy256 = tSetQuerySqlNode(&yymsp[-1].minor.yy0, yymsp[0].minor.yy131, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy185 = yylhsminor.yy185; + yymsp[-1].minor.yy256 = yylhsminor.yy256; break; case 183: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy225 = yymsp[-1].minor.yy225;} - yymsp[-1].minor.yy225 = yylhsminor.yy225; +{yylhsminor.yy131 = yymsp[-1].minor.yy131;} + yymsp[-1].minor.yy131 = yylhsminor.yy131; break; case 184: /* sclp ::= */ case 216: /* orderby_opt ::= */ yytestcase(yyruleno==216); -{yymsp[1].minor.yy225 = 0;} +{yymsp[1].minor.yy131 = 0;} break; case 185: /* selcollist ::= sclp distinct expr as */ { - yylhsminor.yy225 = tSqlExprListAppend(yymsp[-3].minor.yy225, yymsp[-1].minor.yy226, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yylhsminor.yy131 = tSqlExprListAppend(yymsp[-3].minor.yy131, yymsp[-1].minor.yy46, yymsp[-2].minor.yy0.n? &yymsp[-2].minor.yy0:0, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-3].minor.yy225 = yylhsminor.yy225; + yymsp[-3].minor.yy131 = yylhsminor.yy131; break; case 186: /* selcollist ::= sclp STAR */ { tSqlExpr *pNode = tSqlExprCreateIdValue(NULL, TK_ALL); - yylhsminor.yy225 = tSqlExprListAppend(yymsp[-1].minor.yy225, pNode, 0, 0); + yylhsminor.yy131 = tSqlExprListAppend(yymsp[-1].minor.yy131, pNode, 0, 0); } - yymsp[-1].minor.yy225 = yylhsminor.yy225; + yymsp[-1].minor.yy131 = yylhsminor.yy131; break; case 187: /* as ::= AS ids */ { yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } @@ -2874,85 +2849,85 @@ static YYACTIONTYPE yy_reduce( break; case 192: /* from ::= FROM tablelist */ case 193: /* from ::= FROM sub */ yytestcase(yyruleno==193); -{yymsp[-1].minor.yy160 = yymsp[0].minor.yy160;} +{yymsp[-1].minor.yy544 = yymsp[0].minor.yy544;} break; case 194: /* sub ::= LP union RP */ -{yymsp[-2].minor.yy160 = addSubquery(NULL, yymsp[-1].minor.yy93, NULL);} +{yymsp[-2].minor.yy544 = addSubquery(NULL, yymsp[-1].minor.yy303, NULL);} break; case 195: /* sub ::= LP union RP ids */ -{yymsp[-3].minor.yy160 = addSubquery(NULL, yymsp[-2].minor.yy93, &yymsp[0].minor.yy0);} +{yymsp[-3].minor.yy544 = addSubquery(NULL, yymsp[-2].minor.yy303, &yymsp[0].minor.yy0);} break; case 196: /* sub ::= sub COMMA LP union RP ids */ -{yylhsminor.yy160 = addSubquery(yymsp[-5].minor.yy160, yymsp[-2].minor.yy93, &yymsp[0].minor.yy0);} - yymsp[-5].minor.yy160 = yylhsminor.yy160; +{yylhsminor.yy544 = addSubquery(yymsp[-5].minor.yy544, yymsp[-2].minor.yy303, &yymsp[0].minor.yy0);} + yymsp[-5].minor.yy544 = yylhsminor.yy544; break; case 197: /* tablelist ::= ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy160 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); + yylhsminor.yy544 = setTableNameList(NULL, &yymsp[-1].minor.yy0, NULL); } - yymsp[-1].minor.yy160 = yylhsminor.yy160; + yymsp[-1].minor.yy544 = yylhsminor.yy544; break; case 198: /* tablelist ::= ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy160 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy544 = setTableNameList(NULL, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy160 = yylhsminor.yy160; + yymsp[-2].minor.yy544 = yylhsminor.yy544; break; case 199: /* tablelist ::= tablelist COMMA ids cpxName */ { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy160 = setTableNameList(yymsp[-3].minor.yy160, &yymsp[-1].minor.yy0, NULL); + yylhsminor.yy544 = setTableNameList(yymsp[-3].minor.yy544, &yymsp[-1].minor.yy0, NULL); } - yymsp[-3].minor.yy160 = yylhsminor.yy160; + yymsp[-3].minor.yy544 = yylhsminor.yy544; break; case 200: /* tablelist ::= tablelist COMMA ids cpxName ids */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy160 = setTableNameList(yymsp[-4].minor.yy160, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy544 = setTableNameList(yymsp[-4].minor.yy544, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } - yymsp[-4].minor.yy160 = yylhsminor.yy160; + yymsp[-4].minor.yy544 = yylhsminor.yy544; break; case 201: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; case 202: /* interval_option ::= intervalKey LP tmvar RP */ -{yylhsminor.yy52.interval = yymsp[-1].minor.yy0; yylhsminor.yy52.offset.n = 0; yylhsminor.yy52.token = yymsp[-3].minor.yy460;} - yymsp[-3].minor.yy52 = yylhsminor.yy52; +{yylhsminor.yy530.interval = yymsp[-1].minor.yy0; yylhsminor.yy530.offset.n = 0; yylhsminor.yy530.token = yymsp[-3].minor.yy310;} + yymsp[-3].minor.yy530 = yylhsminor.yy530; break; case 203: /* interval_option ::= intervalKey LP tmvar COMMA tmvar RP */ -{yylhsminor.yy52.interval = yymsp[-3].minor.yy0; yylhsminor.yy52.offset = yymsp[-1].minor.yy0; yylhsminor.yy52.token = yymsp[-5].minor.yy460;} - yymsp[-5].minor.yy52 = yylhsminor.yy52; +{yylhsminor.yy530.interval = yymsp[-3].minor.yy0; yylhsminor.yy530.offset = yymsp[-1].minor.yy0; yylhsminor.yy530.token = yymsp[-5].minor.yy310;} + yymsp[-5].minor.yy530 = yylhsminor.yy530; break; case 204: /* interval_option ::= */ -{memset(&yymsp[1].minor.yy52, 0, sizeof(yymsp[1].minor.yy52));} +{memset(&yymsp[1].minor.yy530, 0, sizeof(yymsp[1].minor.yy530));} break; case 205: /* intervalKey ::= INTERVAL */ -{yymsp[0].minor.yy460 = TK_INTERVAL;} +{yymsp[0].minor.yy310 = TK_INTERVAL;} break; case 206: /* intervalKey ::= EVERY */ -{yymsp[0].minor.yy460 = TK_EVERY; } +{yymsp[0].minor.yy310 = TK_EVERY; } break; case 207: /* session_option ::= */ -{yymsp[1].minor.yy463.col.n = 0; yymsp[1].minor.yy463.gap.n = 0;} +{yymsp[1].minor.yy39.col.n = 0; yymsp[1].minor.yy39.gap.n = 0;} break; case 208: /* session_option ::= SESSION LP ids cpxName COMMA tmvar RP */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - yymsp[-6].minor.yy463.col = yymsp[-4].minor.yy0; - yymsp[-6].minor.yy463.gap = yymsp[-1].minor.yy0; + yymsp[-6].minor.yy39.col = yymsp[-4].minor.yy0; + yymsp[-6].minor.yy39.gap = yymsp[-1].minor.yy0; } break; case 209: /* windowstate_option ::= */ -{ yymsp[1].minor.yy112.col.n = 0; yymsp[1].minor.yy112.col.z = NULL;} +{ yymsp[1].minor.yy538.col.n = 0; yymsp[1].minor.yy538.col.z = NULL;} break; case 210: /* windowstate_option ::= STATE_WINDOW LP ids RP */ -{ yymsp[-3].minor.yy112.col = yymsp[-1].minor.yy0; } +{ yymsp[-3].minor.yy538.col = yymsp[-1].minor.yy0; } break; case 211: /* fill_opt ::= */ -{ yymsp[1].minor.yy225 = 0; } +{ yymsp[1].minor.yy131 = 0; } break; case 212: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { @@ -2960,14 +2935,14 @@ static YYACTIONTYPE yy_reduce( toTSDBType(yymsp[-3].minor.yy0.type); taosVariantCreate(&A, yymsp[-3].minor.yy0.z, yymsp[-3].minor.yy0.n, yymsp[-3].minor.yy0.type); - tListItemInsert(yymsp[-1].minor.yy225, &A, -1, 0); - yymsp[-5].minor.yy225 = yymsp[-1].minor.yy225; + tListItemInsert(yymsp[-1].minor.yy131, &A, -1, 0); + yymsp[-5].minor.yy131 = yymsp[-1].minor.yy131; } break; case 213: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy225 = tListItemAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yymsp[-3].minor.yy131 = tListItemAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; case 214: /* sliding_opt ::= SLIDING LP tmvar RP */ @@ -2977,243 +2952,243 @@ static YYACTIONTYPE yy_reduce( {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; case 217: /* orderby_opt ::= ORDER BY sortlist */ -{yymsp[-2].minor.yy225 = yymsp[0].minor.yy225;} +{yymsp[-2].minor.yy131 = yymsp[0].minor.yy131;} break; case 218: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy225 = tListItemAppend(yymsp[-3].minor.yy225, &yymsp[-1].minor.yy1, yymsp[0].minor.yy40); + yylhsminor.yy131 = tListItemAppend(yymsp[-3].minor.yy131, &yymsp[-1].minor.yy43, yymsp[0].minor.yy44); } - yymsp[-3].minor.yy225 = yylhsminor.yy225; + yymsp[-3].minor.yy131 = yylhsminor.yy131; break; case 219: /* sortlist ::= item sortorder */ { - yylhsminor.yy225 = tListItemAppend(NULL, &yymsp[-1].minor.yy1, yymsp[0].minor.yy40); + yylhsminor.yy131 = tListItemAppend(NULL, &yymsp[-1].minor.yy43, yymsp[0].minor.yy44); } - yymsp[-1].minor.yy225 = yylhsminor.yy225; + yymsp[-1].minor.yy131 = yylhsminor.yy131; break; case 220: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - taosVariantCreate(&yylhsminor.yy1, yymsp[-1].minor.yy0.z, yymsp[-1].minor.yy0.n, yymsp[-1].minor.yy0.type); + taosVariantCreate(&yylhsminor.yy43, yymsp[-1].minor.yy0.z, yymsp[-1].minor.yy0.n, yymsp[-1].minor.yy0.type); } - yymsp[-1].minor.yy1 = yylhsminor.yy1; + yymsp[-1].minor.yy43 = yylhsminor.yy43; break; case 221: /* sortorder ::= ASC */ -{ yymsp[0].minor.yy40 = TSDB_ORDER_ASC; } +{ yymsp[0].minor.yy44 = TSDB_ORDER_ASC; } break; case 222: /* sortorder ::= DESC */ -{ yymsp[0].minor.yy40 = TSDB_ORDER_DESC;} +{ yymsp[0].minor.yy44 = TSDB_ORDER_DESC;} break; case 223: /* sortorder ::= */ -{ yymsp[1].minor.yy40 = TSDB_ORDER_ASC; } +{ yymsp[1].minor.yy44 = TSDB_ORDER_ASC; } break; case 224: /* groupby_opt ::= */ -{ yymsp[1].minor.yy225 = 0;} +{ yymsp[1].minor.yy131 = 0;} break; case 225: /* groupby_opt ::= GROUP BY grouplist */ -{ yymsp[-2].minor.yy225 = yymsp[0].minor.yy225;} +{ yymsp[-2].minor.yy131 = yymsp[0].minor.yy131;} break; case 226: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy225 = tListItemAppend(yymsp[-2].minor.yy225, &yymsp[0].minor.yy1, -1); + yylhsminor.yy131 = tListItemAppend(yymsp[-2].minor.yy131, &yymsp[0].minor.yy43, -1); } - yymsp[-2].minor.yy225 = yylhsminor.yy225; + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 227: /* grouplist ::= item */ { - yylhsminor.yy225 = tListItemAppend(NULL, &yymsp[0].minor.yy1, -1); + yylhsminor.yy131 = tListItemAppend(NULL, &yymsp[0].minor.yy43, -1); } - yymsp[0].minor.yy225 = yylhsminor.yy225; + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 228: /* having_opt ::= */ case 238: /* where_opt ::= */ yytestcase(yyruleno==238); case 282: /* expritem ::= */ yytestcase(yyruleno==282); -{yymsp[1].minor.yy226 = 0;} +{yymsp[1].minor.yy46 = 0;} break; case 229: /* having_opt ::= HAVING expr */ case 239: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==239); -{yymsp[-1].minor.yy226 = yymsp[0].minor.yy226;} +{yymsp[-1].minor.yy46 = yymsp[0].minor.yy46;} break; case 230: /* limit_opt ::= */ case 234: /* slimit_opt ::= */ yytestcase(yyruleno==234); -{yymsp[1].minor.yy495.limit = -1; yymsp[1].minor.yy495.offset = 0;} +{yymsp[1].minor.yy55.limit = -1; yymsp[1].minor.yy55.offset = 0;} break; case 231: /* limit_opt ::= LIMIT signed */ case 235: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==235); -{yymsp[-1].minor.yy495.limit = yymsp[0].minor.yy61; yymsp[-1].minor.yy495.offset = 0;} +{yymsp[-1].minor.yy55.limit = yymsp[0].minor.yy459; yymsp[-1].minor.yy55.offset = 0;} break; case 232: /* limit_opt ::= LIMIT signed OFFSET signed */ -{ yymsp[-3].minor.yy495.limit = yymsp[-2].minor.yy61; yymsp[-3].minor.yy495.offset = yymsp[0].minor.yy61;} +{ yymsp[-3].minor.yy55.limit = yymsp[-2].minor.yy459; yymsp[-3].minor.yy55.offset = yymsp[0].minor.yy459;} break; case 233: /* limit_opt ::= LIMIT signed COMMA signed */ -{ yymsp[-3].minor.yy495.limit = yymsp[0].minor.yy61; yymsp[-3].minor.yy495.offset = yymsp[-2].minor.yy61;} +{ yymsp[-3].minor.yy55.limit = yymsp[0].minor.yy459; yymsp[-3].minor.yy55.offset = yymsp[-2].minor.yy459;} break; case 236: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ -{yymsp[-3].minor.yy495.limit = yymsp[-2].minor.yy61; yymsp[-3].minor.yy495.offset = yymsp[0].minor.yy61;} +{yymsp[-3].minor.yy55.limit = yymsp[-2].minor.yy459; yymsp[-3].minor.yy55.offset = yymsp[0].minor.yy459;} break; case 237: /* slimit_opt ::= SLIMIT signed COMMA signed */ -{yymsp[-3].minor.yy495.limit = yymsp[0].minor.yy61; yymsp[-3].minor.yy495.offset = yymsp[-2].minor.yy61;} +{yymsp[-3].minor.yy55.limit = yymsp[0].minor.yy459; yymsp[-3].minor.yy55.offset = yymsp[-2].minor.yy459;} break; case 240: /* expr ::= LP expr RP */ -{yylhsminor.yy226 = yymsp[-1].minor.yy226; yylhsminor.yy226->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy226->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = yymsp[-1].minor.yy46; yylhsminor.yy46->exprToken.z = yymsp[-2].minor.yy0.z; yylhsminor.yy46->exprToken.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 241: /* expr ::= ID */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_ID);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 242: /* expr ::= ID DOT ID */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ID);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 243: /* expr ::= ID DOT STAR */ -{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{ yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[-2].minor.yy0, TK_ALL);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 244: /* expr ::= INTEGER */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_INTEGER);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 245: /* expr ::= MINUS INTEGER */ case 246: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==246); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy226 = yylhsminor.yy226; +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_INTEGER);} + yymsp[-1].minor.yy46 = yylhsminor.yy46; break; case 247: /* expr ::= FLOAT */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_FLOAT);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 248: /* expr ::= MINUS FLOAT */ case 249: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==249); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy226 = yylhsminor.yy226; +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_FLOAT);} + yymsp[-1].minor.yy46 = yylhsminor.yy46; break; case 250: /* expr ::= STRING */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_STRING);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 251: /* expr ::= NOW */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NOW); } + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 252: /* expr ::= VARIABLE */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_VARIABLE);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 253: /* expr ::= PLUS VARIABLE */ case 254: /* expr ::= MINUS VARIABLE */ yytestcase(yyruleno==254); -{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} - yymsp[-1].minor.yy226 = yylhsminor.yy226; +{ yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_VARIABLE; yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[-1].minor.yy0, TK_VARIABLE);} + yymsp[-1].minor.yy46 = yylhsminor.yy46; break; case 255: /* expr ::= BOOL */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 256: /* expr ::= NULL */ -{ yylhsminor.yy226 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{ yylhsminor.yy46 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 257: /* expr ::= ID LP exprlist RP */ -{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy226 = tSqlExprCreateFunction(yymsp[-1].minor.yy225, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy226 = yylhsminor.yy226; +{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy46 = tSqlExprCreateFunction(yymsp[-1].minor.yy131, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy46 = yylhsminor.yy46; break; case 258: /* expr ::= ID LP STAR RP */ -{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy226 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy226 = yylhsminor.yy226; +{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy46 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } + yymsp[-3].minor.yy46 = yylhsminor.yy46; break; case 259: /* expr ::= expr IS NULL */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, NULL, TK_ISNULL);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, NULL, TK_ISNULL);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 260: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-3].minor.yy226, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-3].minor.yy46, NULL, TK_NOTNULL);} + yymsp[-3].minor.yy46 = yylhsminor.yy46; break; case 261: /* expr ::= expr LT expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_LT);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_LT);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 262: /* expr ::= expr GT expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_GT);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_GT);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 263: /* expr ::= expr LE expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_LE);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_LE);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 264: /* expr ::= expr GE expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_GE);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_GE);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 265: /* expr ::= expr NE expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_NE);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_NE);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 266: /* expr ::= expr EQ expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_EQ);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_EQ);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 267: /* expr ::= expr BETWEEN expr AND expr */ -{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy226); yylhsminor.yy226 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy226, yymsp[-2].minor.yy226, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy226, TK_LE), TK_AND);} - yymsp[-4].minor.yy226 = yylhsminor.yy226; +{ tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy46); yylhsminor.yy46 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy46, yymsp[-2].minor.yy46, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy46, TK_LE), TK_AND);} + yymsp[-4].minor.yy46 = yylhsminor.yy46; break; case 268: /* expr ::= expr AND expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_AND);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_AND);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 269: /* expr ::= expr OR expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_OR); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_OR); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 270: /* expr ::= expr PLUS expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_PLUS); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_PLUS); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 271: /* expr ::= expr MINUS expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_MINUS); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_MINUS); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 272: /* expr ::= expr STAR expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_STAR); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_STAR); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 273: /* expr ::= expr SLASH expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_DIVIDE);} - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_DIVIDE);} + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 274: /* expr ::= expr REM expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_REM); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_REM); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 275: /* expr ::= expr LIKE expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_LIKE); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_LIKE); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 276: /* expr ::= expr MATCH expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_MATCH); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_MATCH); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 277: /* expr ::= expr NMATCH expr */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-2].minor.yy226, yymsp[0].minor.yy226, TK_NMATCH); } - yymsp[-2].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-2].minor.yy46, yymsp[0].minor.yy46, TK_NMATCH); } + yymsp[-2].minor.yy46 = yylhsminor.yy46; break; case 278: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy226 = tSqlExprCreate(yymsp[-4].minor.yy226, (tSqlExpr*)yymsp[-1].minor.yy225, TK_IN); } - yymsp[-4].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = tSqlExprCreate(yymsp[-4].minor.yy46, (tSqlExpr*)yymsp[-1].minor.yy131, TK_IN); } + yymsp[-4].minor.yy46 = yylhsminor.yy46; break; case 279: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy225 = tSqlExprListAppend(yymsp[-2].minor.yy225,yymsp[0].minor.yy226,0, 0);} - yymsp[-2].minor.yy225 = yylhsminor.yy225; +{yylhsminor.yy131 = tSqlExprListAppend(yymsp[-2].minor.yy131,yymsp[0].minor.yy46,0, 0);} + yymsp[-2].minor.yy131 = yylhsminor.yy131; break; case 280: /* exprlist ::= expritem */ -{yylhsminor.yy225 = tSqlExprListAppend(0,yymsp[0].minor.yy226,0, 0);} - yymsp[0].minor.yy225 = yylhsminor.yy225; +{yylhsminor.yy131 = tSqlExprListAppend(0,yymsp[0].minor.yy46,0, 0);} + yymsp[0].minor.yy131 = yylhsminor.yy131; break; case 281: /* expritem ::= expr */ -{yylhsminor.yy226 = yymsp[0].minor.yy226;} - yymsp[0].minor.yy226 = yylhsminor.yy226; +{yylhsminor.yy46 = yymsp[0].minor.yy46;} + yymsp[0].minor.yy46 = yylhsminor.yy46; break; case 283: /* cmd ::= RESET QUERY CACHE */ { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} @@ -3224,7 +3199,7 @@ static YYACTIONTYPE yy_reduce( case 285: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3240,14 +3215,14 @@ static YYACTIONTYPE yy_reduce( case 287: /* cmd ::= ALTER TABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; case 288: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_TAG, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3282,7 +3257,7 @@ static YYACTIONTYPE yy_reduce( toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tListItemAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tListItemAppend(A, &yymsp[0].minor.yy1, -1); + A = tListItemAppend(A, &yymsp[0].minor.yy43, -1); SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); @@ -3291,14 +3266,14 @@ static YYACTIONTYPE yy_reduce( case 292: /* cmd ::= ALTER TABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; case 293: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3316,14 +3291,14 @@ static YYACTIONTYPE yy_reduce( case 295: /* cmd ::= ALTER STABLE ids cpxName MODIFY COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; case 296: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_ADD_TAG, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3358,7 +3333,7 @@ static YYACTIONTYPE yy_reduce( toTSDBType(yymsp[-2].minor.yy0.type); SArray* A = tListItemAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tListItemAppend(A, &yymsp[0].minor.yy1, -1); + A = tListItemAppend(A, &yymsp[0].minor.yy43, -1); SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); @@ -3367,7 +3342,7 @@ static YYACTIONTYPE yy_reduce( case 300: /* cmd ::= ALTER STABLE ids cpxName MODIFY TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy225, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); + SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy131, NULL, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; @@ -3401,7 +3376,6 @@ static YYACTIONTYPE yy_reduce( yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; } /* @@ -3411,8 +3385,7 @@ static YYACTIONTYPE yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); @@ -3423,8 +3396,7 @@ static void yy_parse_failed( ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -3436,8 +3408,7 @@ static void yy_syntax_error( int yymajor, /* The major type of the error token */ ParseTOKENTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ @@ -3463,8 +3434,7 @@ static void yy_syntax_error( assert(len <= outputBufLen); /************ End %syntax_error code ******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* @@ -3473,8 +3443,7 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); @@ -3489,8 +3458,7 @@ static void yy_accept( /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* The main parser program. @@ -3519,47 +3487,45 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ + unsigned int yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - ParseCTX_FETCH - ParseARG_STORE + yyParser *yypParser; /* The parser */ + yypParser = (yyParser*)yyp; assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif + ParseARG_STORE; - yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ + int stateno = yypParser->yytos->stateno; + if( stateno < YY_MIN_REDUCE ){ fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); + yyTracePrompt,yyTokenName[yymajor],stateno); }else{ fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE); } } #endif do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor ParseCTX_PARAM); + yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); + yy_shift(yypParser,yyact,yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif - break; + yymajor = YYNOCODE; }else if( yyact==YY_ACCEPT_ACTION ){ yypParser->yytos--; yy_accept(yypParser); @@ -3610,9 +3576,10 @@ void Parse( yymajor = YYNOCODE; }else{ while( yypParser->yytos >= yypParser->yystack + && yymx != YYERRORSYMBOL && (yyact = yy_find_reduce_action( yypParser->yytos->stateno, - YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + YYERRORSYMBOL)) >= YY_MIN_REDUCE ){ yy_pop_parser_stack(yypParser); } @@ -3629,8 +3596,6 @@ void Parse( } yypParser->yyerrcnt = 3; yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3641,7 +3606,8 @@ void Parse( */ yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; + yymajor = YYNOCODE; + #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3663,10 +3629,10 @@ void Parse( yypParser->yyerrcnt = -1; #endif } - break; + yymajor = YYNOCODE; #endif } - }while( yypParser->yytos>yypParser->yystack ); + }while( yymajor!=YYNOCODE && yypParser->yytos>yypParser->yystack ); #ifndef NDEBUG if( yyTraceFILE ){ yyStackEntry *i; @@ -3681,18 +3647,3 @@ void Parse( #endif return; } - -/* -** Return the fallback token corresponding to canonical token iToken, or -** 0 if iToken has no fallback. -*/ -int ParseFallback(int iToken){ -#ifdef YYFALLBACK - if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ - return yyFallback[iToken]; - } -#else - (void)iToken; -#endif - return 0; -} diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 8ed67d7f2ff6bcb67b8f94ceca68b02e102b51e9..457d586ea41acc0161dddd9664eeb2156bc19e46 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -29,9 +29,10 @@ namespace { void generateTestT1(MockCatalogService* mcs) { - ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 4) + ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 6) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) - .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT); + .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT) + .addColumn("c4", TSDB_DATA_TYPE_DOUBLE).addColumn("c5", TSDB_DATA_TYPE_DOUBLE); builder.done(); } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 26ae44a08f876144dc8341717c6cadea7104d96b..248a24f51e9ffd90019f176bf62a4dc7f86ac1c1 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "tarray.h" #include "planner.h" #include "parser.h" diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index eadc95b98d1148e3732188ab12847f4d5162ab16..748be8fcd86d658b5a964884ee2b087fe156eae7 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,7 +14,7 @@ */ #include "plannerInt.h" -#include "exception.h" +#include "texception.h" #include "parser.h" #define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index be570f0b96106f168c067db82165c571d8a9d18e..b7e28d70db2110672401861e42125f48e00497e6 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -55,6 +55,9 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { SNode* pExpr; int32_t index = 0; FOREACH(pExpr, pCxt->pExprs) { + if (QUERY_NODE_GROUPING_SET == nodeType(pExpr)) { + pExpr = nodesListGetNode(((SGroupingSetNode*)pExpr)->pParameterList, 0); + } if (nodesEqualNode(pExpr, *pNode)) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); CHECK_ALLOC(pCol, DEAL_RES_ERROR); @@ -156,6 +159,10 @@ static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect, CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); } + pScan->scanType = SCAN_TYPE_TABLE; + pScan->scanFlag = MAIN_SCAN; + pScan->scanRange = TSWINDOW_INITIALIZER; + return (SLogicNode*)pScan; } @@ -394,34 +401,25 @@ int32_t splitLogicPlan(SSubLogicPlan* pLogicPlan) { } typedef struct SSlotIndex { - int16_t tupleId; + int16_t dataBlockId; int16_t slotId; } SSlotIndex; typedef struct SPhysiPlanContext { int32_t errCode; - int16_t nextTupleId; - SArray* pTupleHelper; + int16_t nextDataBlockId; + SArray* pLocationHelper; } SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - return sprintf(pKey, "%s.%s", ((SColumnNode*)pNode)->tableAlias, ((SColumnNode*)pNode)->colName); - } else { - return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); - } -} - -static SNode* createColumnRef(SNode* pNode, int16_t tupleId, int16_t slotId) { - SColumnRefNode* pCol = (SColumnRefNode*)nodesMakeNode(QUERY_NODE_COLUMN_REF); - if (NULL == pCol) { - return NULL; + SColumnNode* pCol = (SColumnNode*)pNode; + if ('\0' == pCol->tableAlias[0]) { + return sprintf(pKey, "%s", pCol->colName); + } + return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } - pCol->dataType = ((SExprNode*)pNode)->resType; - pCol->tupleId = tupleId; - pCol->slotId = slotId; - pCol->columnId = (QUERY_NODE_COLUMN == nodeType(pNode) ? ((SColumnNode*)pNode)->colId : -1); - return (SNode*)pCol; + return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); } static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_t slotId) { @@ -429,101 +427,92 @@ static SNode* createSlotDesc(SPhysiPlanContext* pCxt, const SNode* pNode, int16_ CHECK_ALLOC(pSlot, NULL); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; - pSlot->srcTupleId = -1; - pSlot->srcSlotId = -1; pSlot->reserve = false; - pSlot->output = true; + pSlot->output = false; return (SNode*)pSlot; } -static SNode* createTarget(SNode* pNode, int16_t tupleId, int16_t slotId) { +static SNode* createTarget(SNode* pNode, int16_t dataBlockId, int16_t slotId) { STargetNode* pTarget = (STargetNode*)nodesMakeNode(QUERY_NODE_TARGET); if (NULL == pTarget) { return NULL; } - pTarget->tupleId = tupleId; + pTarget->dataBlockId = dataBlockId; pTarget->slotId = slotId; - pTarget->pExpr = nodesCloneNode(pNode); - if (NULL == pTarget->pExpr) { - nodesDestroyNode((SNode*)pTarget); - return NULL; - } + pTarget->pExpr = pNode; return (SNode*)pTarget; } -static int32_t addTupleDesc(SPhysiPlanContext* pCxt, SNodeList* pList, STupleDescNode* pTuple, SNodeList** pOutput) { - pTuple->tupleId = pCxt->nextTupleId++; - +static int32_t addDataBlockDesc(SPhysiPlanContext* pCxt, SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { SHashObj* pHash = NULL; - if (NULL == pTuple->pSlots) { - pTuple->pSlots = nodesMakeList(); - CHECK_ALLOC(pTuple->pSlots, TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pDataBlockDesc->pSlots) { + pDataBlockDesc->pSlots = nodesMakeList(); + CHECK_ALLOC(pDataBlockDesc->pSlots, TSDB_CODE_OUT_OF_MEMORY); pHash = taosHashInit(LIST_LENGTH(pList), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); CHECK_ALLOC(pHash, TSDB_CODE_OUT_OF_MEMORY); - if (NULL == taosArrayInsert(pCxt->pTupleHelper, pTuple->tupleId, &pHash)) { + if (NULL == taosArrayInsert(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId, &pHash)) { taosHashCleanup(pHash); return TSDB_CODE_OUT_OF_MEMORY; } } else { - pHash = taosArrayGetP(pCxt->pTupleHelper, pTuple->tupleId); + pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); } - *pOutput = nodesMakeList(); - CHECK_ALLOC(*pOutput, TSDB_CODE_OUT_OF_MEMORY); - SNode* pNode = NULL; - int16_t slotId = 0; + int16_t slotId = taosHashGetSize(pHash); FOREACH(pNode, pList) { SNode* pSlot = createSlotDesc(pCxt, pNode, slotId); CHECK_ALLOC(pSlot, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(pTuple->pSlots, (SNode*)pSlot)) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pDataBlockDesc->pSlots, (SNode*)pSlot)) { nodesDestroyNode(pSlot); return TSDB_CODE_OUT_OF_MEMORY; } - SNode* pTarget = createTarget(pNode, pTuple->tupleId, slotId); - CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); - if (TSDB_CODE_SUCCESS != nodesListAppend(*pOutput, pTarget)) { - nodesDestroyNode(pTarget); - return TSDB_CODE_OUT_OF_MEMORY; - } - - SSlotIndex index = { .tupleId = pTuple->tupleId, .slotId = slotId }; + SSlotIndex index = { .dataBlockId = pDataBlockDesc->dataBlockId, .slotId = slotId }; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; int32_t len = getSlotKey(pNode, name); CHECK_CODE(taosHashPut(pHash, name, len, &index, sizeof(SSlotIndex)), TSDB_CODE_OUT_OF_MEMORY); + SNode* pTarget = createTarget(pNode, pDataBlockDesc->dataBlockId, slotId); + CHECK_ALLOC(pTarget, TSDB_CODE_OUT_OF_MEMORY); + REPLACE_NODE(pTarget); + ++slotId; } return TSDB_CODE_SUCCESS; } -typedef struct STransformCxt { +typedef struct SSetSlotIdCxt { int32_t errCode; - SHashObj* pHash; -} STransformCxt; + SHashObj* pLeftHash; + SHashObj* pRightHash; +} SSetSlotIdCxt; -static EDealRes doTransform(SNode** pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(*pNode)) { - STransformCxt* pCxt = (STransformCxt*)pContext; +static EDealRes doSetSlotId(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode) && 0 != strcmp(((SColumnNode*)pNode)->colName, "*")) { + SSetSlotIdCxt* pCxt = (SSetSlotIdCxt*)pContext; char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; - int32_t len = getSlotKey(*pNode, name); - SSlotIndex* pIndex = taosHashGet(pCxt->pHash, name, len); - if (NULL != pIndex) { - *pNode = createColumnRef(*pNode, pIndex->tupleId, pIndex->slotId); - CHECK_ALLOC(*pNode, DEAL_RES_ERROR); - return DEAL_RES_IGNORE_CHILD; + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pCxt->pLeftHash, name, len); + if (NULL == pIndex) { + pIndex = taosHashGet(pCxt->pRightHash, name, len); } + // pIndex is definitely not NULL, otherwise it is a bug + ((SColumnNode*)pNode)->dataBlockId = pIndex->dataBlockId; + ((SColumnNode*)pNode)->slotId = pIndex->slotId; + CHECK_ALLOC(pNode, DEAL_RES_ERROR); + return DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; } -static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNode* pNode) { +static SNode* setNodeSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNode* pNode) { SNode* pRes = nodesCloneNode(pNode); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteNode(&pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + nodesWalkNode(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyNode(pRes); return NULL; @@ -531,11 +520,12 @@ static SNode* transformForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SN return pRes; } -static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tupleId, SNodeList* pList) { +static SNodeList* setListSlotId(SPhysiPlanContext* pCxt, int16_t leftDataBlockId, int16_t rightDataBlockId, SNodeList* pList) { SNodeList* pRes = nodesCloneList(pList); CHECK_ALLOC(pRes, NULL); - STransformCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pHash = taosArrayGetP(pCxt->pTupleHelper, tupleId) }; - nodesRewriteList(pRes, doTransform, &cxt); + SSetSlotIdCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pLeftHash = taosArrayGetP(pCxt->pLocationHelper, leftDataBlockId), + .pRightHash = (rightDataBlockId < 0 ? NULL : taosArrayGetP(pCxt->pLocationHelper, rightDataBlockId)) }; + nodesWalkList(pRes, doSetSlotId, &cxt); if (TSDB_CODE_SUCCESS != cxt.errCode) { nodesDestroyList(pRes); return NULL; @@ -543,22 +533,48 @@ static SNodeList* transformListForPhysiPlan(SPhysiPlanContext* pCxt, int16_t tup return pRes; } -static SPhysiNode* makePhysiNode(ENodeType type) { +static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, ENodeType type) { SPhysiNode* pPhysiNode = (SPhysiNode*)nodesMakeNode(type); if (NULL == pPhysiNode) { return NULL; } - pPhysiNode->outputTuple.type = QUERY_NODE_TUPLE_DESC; + pPhysiNode->outputDataBlockDesc.dataBlockId = pCxt->nextDataBlockId++; + pPhysiNode->outputDataBlockDesc.type = QUERY_NODE_DATABLOCK_DESC; return pPhysiNode; } -static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { - CHECK_CODE(addTupleDesc(pCxt, pScanLogicNode->pScanCols, &pScanPhysiNode->node.outputTuple, &pScanPhysiNode->pScanCols), TSDB_CODE_OUT_OF_MEMORY); +static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pLogicNode, SPhysiNode* pPhysiNode) { + if (NULL != pLogicNode->pConditions) { + pPhysiNode->pConditions = setNodeSlotId(pCxt, pPhysiNode->outputDataBlockDesc.dataBlockId, -1, pLogicNode->pConditions); + CHECK_ALLOC(pPhysiNode->pConditions, TSDB_CODE_OUT_OF_MEMORY); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t setSlotOutput(SPhysiPlanContext* pCxt, SNodeList* pTargets, SDataBlockDescNode* pDataBlockDesc) { + SHashObj* pHash = taosArrayGetP(pCxt->pLocationHelper, pDataBlockDesc->dataBlockId); + char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN]; + SNode* pNode; + FOREACH(pNode, pTargets) { + int32_t len = getSlotKey(pNode, name); + SSlotIndex* pIndex = taosHashGet(pHash, name, len); + ((SSlotDescNode*)nodesListGetNode(pDataBlockDesc->pSlots, pIndex->slotId))->output = true; + } + + return TSDB_CODE_SUCCESS; +} - if (NULL != pScanLogicNode->node.pConditions) { - pScanPhysiNode->node.pConditions = transformForPhysiPlan(pCxt, pScanPhysiNode->node.outputTuple.tupleId, pScanLogicNode->node.pConditions); - CHECK_ALLOC(pScanPhysiNode->node.pConditions, TSDB_CODE_OUT_OF_MEMORY); +static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode, SScanPhysiNode* pScanPhysiNode) { + if (NULL != pScanLogicNode->pScanCols) { + pScanPhysiNode->pScanCols = nodesCloneList(pScanLogicNode->pScanCols); + CHECK_ALLOC(pScanPhysiNode->pScanCols, TSDB_CODE_OUT_OF_MEMORY); } + // Data block describe also needs to be set without scanning column, such as SELECT COUNT(*) FROM t + CHECK_CODE(addDataBlockDesc(pCxt, pScanPhysiNode->pScanCols, &pScanPhysiNode->node.outputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pScanLogicNode, (SPhysiNode*)pScanPhysiNode), TSDB_CODE_OUT_OF_MEMORY); + + CHECK_CODE(setSlotOutput(pCxt, pScanLogicNode->node.pTargets, &pScanPhysiNode->node.outputDataBlockDesc), TSDB_CODE_OUT_OF_MEMORY); pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; @@ -570,14 +586,14 @@ static int32_t initScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanL } static SPhysiNode* createTagScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); + STagScanPhysiNode* pTagScan = (STagScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN); CHECK_ALLOC(pTagScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTagScan), (SPhysiNode*)pTagScan); return (SPhysiNode*)pTagScan; } static SPhysiNode* createTableScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* pScanLogicNode) { - STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); + STableScanPhysiNode* pTableScan = (STableScanPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN); CHECK_ALLOC(pTableScan, NULL); CHECK_CODE(initScanPhysiNode(pCxt, pScanLogicNode, (SScanPhysiNode*)pTableScan), (SPhysiNode*)pTableScan); pTableScan->scanFlag = pScanLogicNode->scanFlag; @@ -597,35 +613,210 @@ static SPhysiNode* createScanPhysiNode(SPhysiPlanContext* pCxt, SScanLogicNode* default: break; } + return NULL; } -static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SProjectLogicNode* pProjectLogicNode) { - SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(QUERY_NODE_PHYSICAL_PLAN_PROJECT); - CHECK_ALLOC(pProject, NULL); +static SNodeList* createJoinOutputCols(SPhysiPlanContext* pCxt, SDataBlockDescNode* pLeftDesc, SDataBlockDescNode* pRightDesc) { + SNodeList* pCols = nodesMakeList(); + CHECK_ALLOC(pCols, NULL); + SNode* pNode; + FOREACH(pNode, pLeftDesc->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = pLeftDesc->dataBlockId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + FOREACH(pNode, pRightDesc->pSlots) { + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + goto error; + } + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = pRightDesc->dataBlockId; + pCol->slotId = pSlot->slotId; + pCol->colId = -1; + if (TSDB_CODE_SUCCESS != nodesListAppend(pCols, (SNode*)pCol)) { + goto error; + } + } + return pCols; +error: + nodesDestroyList(pCols); + return NULL; +} + +static SPhysiNode* createJoinPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SJoinLogicNode* pJoinLogicNode) { + SJoinPhysiNode* pJoin = (SJoinPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_JOIN); + CHECK_ALLOC(pJoin, NULL); + + SDataBlockDescNode* pLeftDesc = &((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc; + SDataBlockDescNode* pRightDesc = &((SPhysiNode*)nodesListGetNode(pChildren, 1))->outputDataBlockDesc; + pJoin->pOnConditions = setNodeSlotId(pCxt, pLeftDesc->dataBlockId, pRightDesc->dataBlockId, pJoinLogicNode->pOnConditions); + CHECK_ALLOC(pJoin->pOnConditions, (SPhysiNode*)pJoin); - SNodeList* pProjections = transformListForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->pProjections); - CHECK_ALLOC(pProjections, (SPhysiNode*)pProject); - CHECK_CODE(addTupleDesc(pCxt, pProjections, &pProject->node.outputTuple, &pProject->pProjections), (SPhysiNode*)pProject); - nodesDestroyList(pProjections); + pJoin->pTargets = createJoinOutputCols(pCxt, pLeftDesc, pRightDesc); + CHECK_ALLOC(pJoin->pTargets, (SPhysiNode*)pJoin); + CHECK_CODE(addDataBlockDesc(pCxt, pJoin->pTargets, &pJoin->node.outputDataBlockDesc), (SPhysiNode*)pJoin); - if (NULL != pProjectLogicNode->node.pConditions) { - pProject->node.pConditions = transformForPhysiPlan(pCxt, pProject->node.outputTuple.tupleId, pProjectLogicNode->node.pConditions); - CHECK_ALLOC(pProject->node.pConditions, (SPhysiNode*)pProject); + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pJoinLogicNode, (SPhysiNode*)pJoin), (SPhysiNode*)pJoin); + + CHECK_CODE(setSlotOutput(pCxt, pJoinLogicNode->node.pTargets, &pJoin->node.outputDataBlockDesc), (SPhysiNode*)pJoin); + + return (SPhysiNode*)pJoin; +} + +typedef struct SRewritePrecalcExprsCxt { + int32_t errCode; + int32_t planNodeId; + int32_t rewriteId; + SNodeList* pPrecalcExprs; +} SRewritePrecalcExprsCxt; + +static EDealRes collectAndRewrite(SRewritePrecalcExprsCxt* pCxt, SNode** pNode) { + SNode* pExpr = nodesCloneNode(*pNode); + CHECK_ALLOC(pExpr, DEAL_RES_ERROR); + if (nodesListAppend(pCxt->pPrecalcExprs, pExpr)) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; } + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + nodesDestroyNode(pExpr); + return DEAL_RES_ERROR; + } + SExprNode* pRewrittenExpr = (SExprNode*)pExpr; + pCol->node.resType = pRewrittenExpr->resType; + if ('\0' != pRewrittenExpr->aliasName[0]) { + strcpy(pCol->colName, pRewrittenExpr->aliasName); + } else { + snprintf(pRewrittenExpr->aliasName, sizeof(pRewrittenExpr->aliasName), "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId); + strcpy(pCol->colName, pRewrittenExpr->aliasName); + } + nodesDestroyNode(*pNode); + *pNode = (SNode*)pCol; + return DEAL_RES_IGNORE_CHILD; +} + +static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) { + SRewritePrecalcExprsCxt* pCxt = (SRewritePrecalcExprsCxt*)pContext; + switch (nodeType(*pNode)) { + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: { + return collectAndRewrite(pContext, pNode); + } + case QUERY_NODE_FUNCTION: { + if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) { + return collectAndRewrite(pContext, pNode); + } + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SNodeList** pPrecalcExprs, SNodeList** pRewrittenList) { + if (NULL == pList) { + return TSDB_CODE_SUCCESS; + } + + if (NULL == *pPrecalcExprs) { + *pPrecalcExprs = nodesMakeList(); + CHECK_ALLOC(*pPrecalcExprs, TSDB_CODE_OUT_OF_MEMORY); + } + if (NULL == *pRewrittenList) { + *pRewrittenList = nodesMakeList(); + CHECK_ALLOC(*pRewrittenList, TSDB_CODE_OUT_OF_MEMORY); + } + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + if (QUERY_NODE_GROUPING_SET == nodeType(pNode)) { + pNew = nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pNode)->pParameterList, 0)); + } else { + pNew = nodesCloneNode(pNode); + } + CHECK_ALLOC(pNew, TSDB_CODE_OUT_OF_MEMORY); + CHECK_CODE(nodesListAppend(*pRewrittenList, pNew), TSDB_CODE_OUT_OF_MEMORY); + } + SRewritePrecalcExprsCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs }; + nodesRewriteList(*pRewrittenList, doRewritePrecalcExprs, &cxt); + if (0 == LIST_LENGTH(cxt.pPrecalcExprs)) { + nodesDestroyList(cxt.pPrecalcExprs); + *pPrecalcExprs = NULL; + } + return cxt.errCode; +} + +static SPhysiNode* createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SAggLogicNode* pAggLogicNode) { + SAggPhysiNode* pAgg = (SAggPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_AGG); + CHECK_ALLOC(pAgg, NULL); + + SNodeList* pPrecalcExprs = NULL; + SNodeList* pGroupKeys = NULL; + SNodeList* pAggFuncs = NULL; + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pGroupKeys, &pPrecalcExprs, &pGroupKeys), (SPhysiNode*)pAgg); + CHECK_CODE(rewritePrecalcExprs(pCxt, pAggLogicNode->pAggFuncs, &pPrecalcExprs, &pAggFuncs), (SPhysiNode*)pAgg); + + SDataBlockDescNode* pChildTupe = &(((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc); + // push down expression to outputDataBlockDesc of child node + if (NULL != pPrecalcExprs) { + pAgg->pExprs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pPrecalcExprs); + CHECK_ALLOC(pAgg->pExprs, (SPhysiNode*)pAgg); + CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pExprs, pChildTupe), (SPhysiNode*)pAgg); + } + + if (NULL != pGroupKeys) { + pAgg->pGroupKeys = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pGroupKeys); + CHECK_ALLOC(pAgg->pGroupKeys, (SPhysiNode*)pAgg); + CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pGroupKeys, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg); + } + + if (NULL != pAggFuncs) { + pAgg->pAggFuncs = setListSlotId(pCxt, pChildTupe->dataBlockId, -1, pAggFuncs); + CHECK_ALLOC(pAgg->pAggFuncs, (SPhysiNode*)pAgg); + CHECK_CODE(addDataBlockDesc(pCxt, pAgg->pAggFuncs, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg); + } + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pAggLogicNode, (SPhysiNode*)pAgg), (SPhysiNode*)pAgg); + + CHECK_CODE(setSlotOutput(pCxt, pAggLogicNode->node.pTargets, &pAgg->node.outputDataBlockDesc), (SPhysiNode*)pAgg); + + return (SPhysiNode*)pAgg; +} + +static SPhysiNode* createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SProjectLogicNode* pProjectLogicNode) { + SProjectPhysiNode* pProject = (SProjectPhysiNode*)makePhysiNode(pCxt, QUERY_NODE_PHYSICAL_PLAN_PROJECT); + CHECK_ALLOC(pProject, NULL); + + pProject->pProjections = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->outputDataBlockDesc.dataBlockId, -1, pProjectLogicNode->pProjections); + CHECK_ALLOC(pProject->pProjections, (SPhysiNode*)pProject); + CHECK_CODE(addDataBlockDesc(pCxt, pProject->pProjections, &pProject->node.outputDataBlockDesc), (SPhysiNode*)pProject); + + CHECK_CODE(setConditionsSlotId(pCxt, (const SLogicNode*)pProjectLogicNode, (SPhysiNode*)pProject), (SPhysiNode*)pProject); return (SPhysiNode*)pProject; } static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPlan) { - SNodeList* pChildern = nodesMakeList(); - CHECK_ALLOC(pChildern, NULL); + SNodeList* pChildren = nodesMakeList(); + CHECK_ALLOC(pChildren, NULL); SNode* pLogicChild; FOREACH(pLogicChild, pLogicPlan->pChildren) { SNode* pChildPhyNode = (SNode*)createPhysiNode(pCxt, (SLogicNode*)pLogicChild); - if (TSDB_CODE_SUCCESS != nodesListAppend(pChildern, pChildPhyNode)) { + if (TSDB_CODE_SUCCESS != nodesListAppend(pChildren, pChildPhyNode)) { pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; - nodesDestroyList(pChildern); + nodesDestroyList(pChildren); return NULL; } } @@ -636,30 +827,30 @@ static SPhysiNode* createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicPl pPhyNode = createScanPhysiNode(pCxt, (SScanLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_JOIN: + pPhyNode = createJoinPhysiNode(pCxt, pChildren, (SJoinLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_AGG: + pPhyNode = createAggPhysiNode(pCxt, pChildren, (SAggLogicNode*)pLogicPlan); break; case QUERY_NODE_LOGIC_PLAN_PROJECT: - pPhyNode = createProjectPhysiNode(pCxt, (SProjectLogicNode*)pLogicPlan); + pPhyNode = createProjectPhysiNode(pCxt, pChildren, (SProjectLogicNode*)pLogicPlan); break; default: break; } - if (NULL != pPhyNode) { - pPhyNode->pChildren = pChildern; - SNode* pChild; - FOREACH(pChild, pPhyNode->pChildren) { - ((SPhysiNode*)pChild)->pParent = pPhyNode; - } + pPhyNode->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, pPhyNode->pChildren) { + ((SPhysiNode*)pChild)->pParent = pPhyNode; } return pPhyNode; } int32_t createPhysiPlan(SLogicNode* pLogicNode, SPhysiNode** pPhyNode) { - SPhysiPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .nextTupleId = 0, .pTupleHelper = taosArrayInit(32, POINTER_BYTES) }; - if (NULL == cxt.pTupleHelper) { + SPhysiPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .nextDataBlockId = 0, .pLocationHelper = taosArrayInit(32, POINTER_BYTES) }; + if (NULL == cxt.pLocationHelper) { return TSDB_CODE_OUT_OF_MEMORY; } *pPhyNode = createPhysiNode(&cxt, pLogicNode); diff --git a/source/libs/planner/test/newPlannerTest.cpp b/source/libs/planner/test/newPlannerTest.cpp index e99f0c150c37a33abacac56435e116bc5ae765f9..e7c0c2483005c0fb8485526c4ad9621841b1f6c9 100644 --- a/source/libs/planner/test/newPlannerTest.cpp +++ b/source/libs/planner/test/newPlannerTest.cpp @@ -62,7 +62,7 @@ protected: return false; } - cout << "sql : [" << cxt_.pSql << "]" << endl; + cout << "====================sql : [" << cxt_.pSql << "]" << endl; cout << "syntax test : " << endl; cout << syntaxTreeStr << endl; cout << "unformatted logic plan : " << endl; @@ -132,7 +132,7 @@ TEST_F(NewPlannerTest, groupBy) { bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3"); ASSERT_TRUE(run()); - bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); + bind("SELECT c1 + c3, sum(c4 * c5) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3"); ASSERT_TRUE(run()); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 4cd040c238931cb228ffcc4ebb6fb1fd3e781f17..a2165453d55820078292eedb7085f7877f459092 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -85,8 +85,9 @@ static void* pTaskQueue = NULL; int32_t initTaskQueue() { double factor = 4.0; - int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t numOfThreads = TMAX((int)(tsNumOfCores * tsNumOfThreadsPerCore / factor), 2); + int32_t queueSize = tsMaxConnections * 2; pTaskQueue = taosInitScheduler(queueSize, numOfThreads, "tsc"); if (NULL == pTaskQueue) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 753ac75860539574f6691dea3e500c241f7dc723..8bc29656d41300a9db7c7ba6d22a22c83c72001d 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -198,7 +198,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl } int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { - int32_t code = -1; + int32_t code = 0; STableMetaRsp metaRsp = {0}; if (NULL == output || NULL == msg || msgSize <= 0) { @@ -216,7 +216,7 @@ int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { goto PROCESS_META_OVER; } - if (!tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { + if (0 != strcmp(metaRsp.dbFName, TSDB_INFORMATION_SCHEMA_DB) && !tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { code = TSDB_CODE_TSC_INVALID_VALUE; goto PROCESS_META_OVER; } @@ -254,11 +254,11 @@ PROCESS_META_OVER: void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; - queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)] = queryProcessTableMetaRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index cebe8178d9303e330bbbd21a211fe34c95ed3b2a..7a6fa4cfd78f1c9278931cfc300a85fc7f4a05f8 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,5 +1,5 @@ #include "qworker.h" -#include +#include "tcommon.h" #include "executor.h" #include "planner.h" #include "query.h" diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index a1b431fd7d92b3b6f1dcb41f6ecb136315527c53..bdb9617d0dc9ae8f44ad2d3d34aa8022307aeea4 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -1,5 +1,5 @@ #include "qworker.h" -#include +#include "tcommon.h" #include "executor.h" #include "planner.h" #include "query.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 38890150ee26ffc1fb092a1a6047c3b2237c1631..8ad5a76388e45c10aee8df05f91629338d7a0f02 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -101,9 +101,7 @@ void qwtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 4700fc6d6efd785e574f48a0315f3812befc8041..b4fe9a67caa2e8bb354d486cd3ff51427266b290 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -22,10 +22,11 @@ extern "C" { #include "thash.h" #include "tname.h" -#include "common.h" +#include "tcommon.h" #include "scalar.h" #include "querynodes.h" #include "query.h" +#include "tep.h" #define FILTER_DEFAULT_GROUP_SIZE 4 #define FILTER_DEFAULT_UNIT_SIZE 4 @@ -199,7 +200,7 @@ typedef struct SFilterUnit { } SFilterUnit; typedef struct SFilterComUnit { - void *colData; + void *colData; // pointer to SColumnInfoData void *valData; void *valData2; uint16_t colId; @@ -252,6 +253,7 @@ typedef struct SFilterInfo { uint32_t *blkUnits; int8_t *blkUnitRes; void *pTable; + SArray *blkList; SFilterPCtx pctx; } SFilterInfo; @@ -282,7 +284,7 @@ typedef struct SFilterInfo { #define INSERT_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r)->prev; if ((r)->prev) { (r)->prev->next = n; } else { (ctx)->rs = n; } (r)->prev = n; n->next = r; } while (0) #define APPEND_RANGE(ctx, r, ra) do { SFilterRangeNode *n = filterNewRange(ctx, ra); n->prev = (r); if (r) { (r)->next = n; } else { (ctx)->rs = n; } } while (0) -#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_NULL) +#define FLT_IS_COMPARISON_OPERATOR(_op) ((_op) >= OP_TYPE_GREATER_THAN && (_op) < OP_TYPE_IS_NOT_UNKNOWN) #define fltFatal(...) qFatal(__VA_ARGS__) #define fltError(...) qError(__VA_ARGS__) @@ -300,12 +302,12 @@ typedef struct SFilterInfo { #define FILTER_GET_FIELD(i, id) (&((i)->fields[(id).type].fields[(id).idx])) #define FILTER_GET_COL_FIELD(i, idx) (&((i)->fields[FLD_TYPE_COLUMN].fields[idx])) -#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.type) -#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnRefNode *)((fi)->desc))->dataType.bytes) -#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnRefNode *)((fi)->desc))->columnId) -#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnRefNode *)((fi)->desc))->slotId) -#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnRefNode *)((fi)->desc)) -#define FILTER_GET_COL_FIELD_DATA(fi, ri) ((char *)(fi)->data + ((SColumnRefNode *)((fi)->desc))->dataType.bytes * (ri)) +#define FILTER_GET_COL_FIELD_TYPE(fi) (((SColumnNode *)((fi)->desc))->node.resType.type) +#define FILTER_GET_COL_FIELD_SIZE(fi) (((SColumnNode *)((fi)->desc))->node.resType.bytes) +#define FILTER_GET_COL_FIELD_ID(fi) (((SColumnNode *)((fi)->desc))->colId) +#define FILTER_GET_COL_FIELD_SLOT_ID(fi) (((SColumnNode *)((fi)->desc))->slotId) +#define FILTER_GET_COL_FIELD_DESC(fi) ((SColumnNode *)((fi)->desc)) +#define FILTER_GET_COL_FIELD_DATA(fi, ri) (colDataGetData(((SColumnInfoData *)(fi)->data), (ri))) #define FILTER_GET_VAL_FIELD_TYPE(fi) (((SValueNode *)((fi)->desc))->node.resType.type) #define FILTER_GET_VAL_FIELD_DATA(fi) ((char *)(fi)->data) #define FILTER_GET_JSON_VAL_FIELD_DATA(fi) ((char *)(fi)->desc) diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 41ee90667e012dde68d9c7179a05b3595001182f..b79221cf82c50bdd560ef40bb8dab90b48c206b4 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -18,13 +18,13 @@ #ifdef __cplusplus extern "C" { #endif -#include "common.h" +#include "tcommon.h" #include "thash.h" #include "query.h" typedef struct SScalarCtx { int32_t code; - SSDataBlock *pSrc; + SArray *pBlockList; /* element is SSDataBlock* */ SHashObj *pRes; /* element is SScalarParam */ } SScalarCtx; @@ -44,10 +44,13 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) - +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx); +bool sclIsNull(SScalarParam* param, int32_t idx); +void sclSetNull(SScalarParam* param, int32_t idx); +void sclFreeParam(SScalarParam *param); #ifdef __cplusplus } #endif -#endif // TDENGINE_SCALARINT_H \ No newline at end of file +#endif // TDENGINE_SCALARINT_H diff --git a/source/libs/scalar/inc/sclvector.h b/source/libs/scalar/inc/sclvector.h index 55c482874511c86ef630c189df751f6b018b71d7..09b813359a6ef98f311577f04b0798dac1d272b4 100644 --- a/source/libs/scalar/inc/sclvector.h +++ b/source/libs/scalar/inc/sclvector.h @@ -22,7 +22,11 @@ extern "C" { #include "sclfunc.h" -typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, void *output, int32_t order); +typedef double (*_mathFunc)(double, double, bool *); + + +typedef void (*_bufConverteFunc)(char *buf, SScalarParam* pOut, int32_t outType); +typedef void (*_bin_scalar_fn_t)(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *output, int32_t order); _bin_scalar_fn_t getBinScalarOperatorFn(int32_t binOperator); #ifdef __cplusplus diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 95b3ea922a5c30d3f3fb995d1d3d7bb059f99a0d..e5be22c0a8862150e15ac0aa6628341e6f5c30c4 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -18,7 +18,9 @@ //#include "queryLog.h" #include "tcompare.h" #include "filterInt.h" +#include "sclInt.h" #include "filter.h" +#include "tep.h" OptrStr gOptrStr[] = { {0, "invalid"}, @@ -914,14 +916,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (nodeType(node) != QUERY_NODE_COLUMN_REF && nodeType(node) != QUERY_NODE_VALUE) { + if (nodeType(node) != QUERY_NODE_COLUMN && nodeType(node) != QUERY_NODE_VALUE && nodeType(node) != QUERY_NODE_NODE_LIST) { FLT_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } int32_t type; void *v; - if (nodeType(node) == QUERY_NODE_COLUMN_REF) { + if (nodeType(node) == QUERY_NODE_COLUMN) { type = FLD_TYPE_COLUMN; v = node; } else { @@ -968,8 +970,10 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); } else { - if(optr != OP_TYPE_IS_NULL && optr != OP_TYPE_IS_NOT_NULL && optr != FILTER_DUMMY_EMPTY_OPTR){ - return -1; + int32_t paramNum = scalarGetOperatorParamNum(optr); + if (1 != paramNum) { + fltError("invalid right field in unit, operator:%s, rightType:%d", gOptrStr[optr].str, u->right.type); + return TSDB_CODE_QRY_APP_ERROR; } } @@ -1016,7 +1020,6 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { if (node->opType == OP_TYPE_IN && (!IS_VAR_DATA_TYPE(type))) { SNodeListNode *listNode = (SNodeListNode *)node->pRight; - void *fdata = NULL; SListCell *cell = listNode->pNodeList->pHead; SScalarParam in = {.num = 1}, out = {.num = 1, .type = type}; @@ -1036,7 +1039,7 @@ int32_t fltAddGroupUnitFromNode(SFilterInfo *info, SNode* tree, SArray *group) { len = tDataTypes[type].bytes; - filterAddField(info, NULL, &fdata, FLD_TYPE_VALUE, &right, len, true); + filterAddField(info, NULL, &out.data, FLD_TYPE_VALUE, &right, len, true); filterAddUnit(info, OP_TYPE_EQUAL, &left, &right, &uidx); @@ -1337,6 +1340,8 @@ EDealRes fltTreeToGroup(SNode* pNode, void* pContext) { return DEAL_RES_IGNORE_CHILD; } + ctx->code = TSDB_CODE_QRY_APP_ERROR; + fltError("invalid condition type, type:%d", node->condType); return DEAL_RES_ERROR; @@ -1451,8 +1456,8 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) qDebug("COLUMN Field Num:%u", info->fields[FLD_TYPE_COLUMN].num); for (uint32_t i = 0; i < info->fields[FLD_TYPE_COLUMN].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_COLUMN].fields[i]; - SColumnRefNode *refNode = (SColumnRefNode *)field->desc; - qDebug("COL%d => [%d][%d]", i, refNode->tupleId, refNode->slotId); + SColumnNode *refNode = (SColumnNode *)field->desc; + qDebug("COL%d => [%d][%d]", i, refNode->dataBlockId, refNode->slotId); } qDebug("VALUE Field Num:%u", info->fields[FLD_TYPE_VALUE].num); @@ -1480,9 +1485,9 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) char str[512] = {0}; SFilterField *left = FILTER_UNIT_LEFT_FIELD(info, unit); - SColumnRefNode *refNode = (SColumnRefNode *)left->desc; + SColumnNode *refNode = (SColumnNode *)left->desc; if (unit->compare.optr >= 0 && unit->compare.optr <= OP_TYPE_JSON_CONTAINS){ - len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr].str); + len = sprintf(str, "UNIT[%d] => [%d][%d] %s [", i, refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr].str); } if (unit->right.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { @@ -1501,7 +1506,7 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) if (unit->compare.optr2) { strcat(str, " && "); if (unit->compare.optr2 >= 0 && unit->compare.optr2 <= OP_TYPE_JSON_CONTAINS){ - sprintf(str + strlen(str), "[%d][%d] %s [", refNode->tupleId, refNode->slotId, gOptrStr[unit->compare.optr2].str); + sprintf(str + strlen(str), "[%d][%d] %s [", refNode->dataBlockId, refNode->slotId, gOptrStr[unit->compare.optr2].str); } if (unit->right2.type == FLD_TYPE_VALUE && FILTER_UNIT_OPTR(unit) != OP_TYPE_IN) { @@ -1674,7 +1679,7 @@ void filterFreeInfo(SFilterInfo *info) { tfree(info->cunits); tfree(info->blkUnitRes); tfree(info->blkUnits); - + for (int32_t i = 0; i < FLD_TYPE_MAX; ++i) { for (uint32_t f = 0; f < info->fields[i].num; ++f) { filterFreeField(&info->fields[i].fields[f], i); @@ -1736,7 +1741,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit* unit = &info->units[i]; if (unit->right.type != FLD_TYPE_VALUE) { - assert(unit->compare.optr == OP_TYPE_IS_NULL || unit->compare.optr == OP_TYPE_IS_NOT_NULL || unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR); + assert(unit->compare.optr == FILTER_DUMMY_EMPTY_OPTR || scalarGetOperatorParamNum(unit->compare.optr) == 1); continue; } @@ -1792,13 +1797,15 @@ int32_t fltInitValFieldData(SFilterInfo *info) { } if(type != TSDB_DATA_TYPE_JSON){ - bool converted = false; - char extInfo = 0; - SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; - SScalarParam out = {.data = fi->data, .num = 1, .type = type}; - if (vectorConvertImpl(&in, &out)) { - qError("convert value to type[%d] failed", type); - return TSDB_CODE_TSC_INVALID_OPERATION; + if (dType->type == type) { + assignVal(fi->data, nodesGetValueFromNode(var), dType->bytes, type); + } else { + SScalarParam in = {.data = nodesGetValueFromNode(var), .num = 1, .type = dType->type, .bytes = dType->bytes}; + SScalarParam out = {.data = fi->data, .num = 1, .type = type}; + if (vectorConvertImpl(&in, &out)) { + qError("convert value to type[%d] failed", type); + return TSDB_CODE_TSC_INVALID_OPERATION; + } } } @@ -1809,7 +1816,7 @@ int32_t fltInitValFieldData(SFilterInfo *info) { int32_t len = taosUcs4ToMbs(varDataVal(fi->data), varDataLen(fi->data), varDataVal(newValData)); if (len < 0){ qError("filterInitValFieldData taosUcs4ToMbs error 1"); - return TSDB_CODE_FAILED; + return TSDB_CODE_QRY_APP_ERROR; } varDataSetLen(newValData, len); varDataCopy(fi->data, newValData); @@ -2565,7 +2572,8 @@ int32_t filterUpdateComUnits(SFilterInfo *info) { for (uint32_t i = 0; i < info->unitNum; ++i) { SFilterUnit *unit = &info->units[i]; - info->cunits[i].colData = FILTER_UNIT_COL_DATA(info, unit, 0); + SFilterField *col = FILTER_UNIT_LEFT_FIELD(info, unit); + info->cunits[i].colData = col->data; } return TSDB_CODE_SUCCESS; @@ -2770,14 +2778,14 @@ bool filterExecuteBasedOnStatisImpl(void *pinfo, int32_t numOfRows, int8_t** p, uint32_t unitNum = *(unitIdx++); for (uint32_t u = 0; u < unitNum; ++u) { SFilterComUnit *cunit = &info->cunits[*(unitIdx + u)]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGetData((SColumnInfoData *)cunit->colData, i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); //} else { uint8_t optr = cunit->optr; - if (isNull(colData, cunit->dataType)) { + if (colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -2868,18 +2876,18 @@ static FORCE_INLINE bool filterExecuteImplIsNull(void *pinfo, int32_t numOfRows, for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData){ // for json->'key' is null (*p)[i] = 1; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ (*p)[i] = 0; } }else{ - (*p)[i] = ((colData == NULL) || isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData == NULL) || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { all = false; @@ -2902,19 +2910,19 @@ static FORCE_INLINE bool filterExecuteImplNotNull(void *pinfo, int32_t numOfRows for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); if(info->cunits[uidx].dataType == TSDB_DATA_TYPE_JSON){ if (!colData) { // for json->'key' is not null (*p)[i] = 0; }else if( *(char*)colData == TSDB_DATA_TYPE_JSON){ // for json is not null colData = POINTER_SHIFT(colData, CHAR_BYTES); - (*p)[i] = !isNull(colData, info->cunits[uidx].dataType); + (*p)[i] = !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL); }else{ // for json->'key' is not null (*p)[i] = 1; } }else { - (*p)[i] = ((colData != NULL) && !isNull(colData, info->cunits[uidx].dataType)); + (*p)[i] = ((colData != NULL) && !colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)); } if ((*p)[i] == 0) { @@ -2929,7 +2937,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD SFilterInfo *info = (SFilterInfo *)pinfo; bool all = true; uint16_t dataSize = info->cunits[0].dataSize; - char *colData = (char *)info->cunits[0].colData; rangeCompFunc rfunc = gRangeCompare[info->cunits[0].rfunc]; void *valData = info->cunits[0].valData; void *valData2 = info->cunits[0].valData2; @@ -2943,10 +2950,11 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD *p = calloc(numOfRows, sizeof(int8_t)); } - for (int32_t i = 0; i < numOfRows; ++i) { - if (colData == NULL || isNull(colData, info->cunits[0].dataType)) { + for (int32_t i = 0; i < numOfRows; ++i) { + void *colData = colDataGetData((SColumnInfoData *)info->cunits[0].colData, i); + + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[0].colData, 0, i, NULL)) { all = false; - colData += dataSize; continue; } @@ -2955,8 +2963,6 @@ bool filterExecuteImplRange(void *pinfo, int32_t numOfRows, int8_t** p, SColumnD if ((*p)[i] == 0) { all = false; } - - colData += dataSize; } return all; @@ -2976,8 +2982,8 @@ bool filterExecuteImplMisc(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDa for (int32_t i = 0; i < numOfRows; ++i) { uint32_t uidx = info->groups[0].unitIdxs[0]; - void *colData = (char *)info->cunits[uidx].colData + info->cunits[uidx].dataSize * i; - if (colData == NULL || isNull(colData, info->cunits[uidx].dataType)) { + void *colData = colDataGetData((SColumnInfoData *)info->cunits[uidx].colData, i); + if (colData == NULL || colDataIsNull((SColumnInfoData *)info->cunits[uidx].colData, 0, i, NULL)) { (*p)[i] = 0; all = false; continue; @@ -3027,14 +3033,14 @@ bool filterExecuteImpl(void *pinfo, int32_t numOfRows, int8_t** p, SColumnDataAg for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; SFilterComUnit *cunit = &info->cunits[uidx]; - void *colData = (char *)cunit->colData + cunit->dataSize * i; + void *colData = colDataGetData((SColumnInfoData *)(cunit->colData), i); //if (FILTER_UNIT_GET_F(info, uidx)) { // p[i] = FILTER_UNIT_GET_R(info, uidx); //} else { uint8_t optr = cunit->optr; - if (colData == NULL || isNull(colData, cunit->dataType)) { + if (colData == NULL || colDataIsNull((SColumnInfoData *)(cunit->colData), 0, i, NULL)) { (*p)[i] = optr == OP_TYPE_IS_NULL ? true : false; } else { if (optr == OP_TYPE_IS_NOT_NULL) { @@ -3483,11 +3489,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (stat->scalarMode) { - return DEAL_RES_CONTINUE; - } - - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN_REF == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } @@ -3505,22 +3507,28 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { if (NULL == node->pRight) { if (scalarGetOperatorParamNum(node->opType) > 1) { - fltError("invalid operator, pRight:%p, type:%d", node->pRight, nodeType(node)); + fltError("invalid operator, pRight:%p, nodeType:%d, opType:%d", node->pRight, nodeType(node), node->opType); stat->code = TSDB_CODE_QRY_APP_ERROR; return DEAL_RES_ERROR; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { + stat->scalarMode = true; + return DEAL_RES_CONTINUE; + } + + if (OP_TYPE_IS_TRUE == node->opType || OP_TYPE_IS_FALSE == node->opType || OP_TYPE_IS_UNKNOWN == node->opType + || OP_TYPE_IS_NOT_TRUE == node->opType || OP_TYPE_IS_NOT_FALSE == node->opType || OP_TYPE_IS_NOT_UNKNOWN == node->opType) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } } else { - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pLeft)) && (QUERY_NODE_VALUE != nodeType(node->pLeft))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } - if ((QUERY_NODE_COLUMN_REF != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { + if ((QUERY_NODE_COLUMN != nodeType(node->pRight)) && (QUERY_NODE_VALUE != nodeType(node->pRight))) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3530,7 +3538,7 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } - if (QUERY_NODE_COLUMN_REF != nodeType(node->pLeft)) { + if (QUERY_NODE_COLUMN != nodeType(node->pLeft)) { SNode *t = node->pLeft; node->pLeft = node->pRight; node->pRight = t; @@ -3543,10 +3551,10 @@ EDealRes fltReviseRewriter(SNode** pNode, void* pContext) { } if (OP_TYPE_IN != node->opType) { - SColumnRefNode *refNode = (SColumnRefNode *)node->pLeft; + SColumnNode *refNode = (SColumnNode *)node->pLeft; SValueNode *valueNode = (SValueNode *)node->pRight; - int32_t type = vectorGetConvertType(refNode->dataType.type, valueNode->node.resType.type); - if (0 != type && type != refNode->dataType.type) { + int32_t type = vectorGetConvertType(refNode->node.resType.type, valueNode->node.resType.type); + if (0 != type && type != refNode->node.resType.type) { stat->scalarMode = true; return DEAL_RES_CONTINUE; } @@ -3570,18 +3578,19 @@ int32_t fltReviseNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { } int32_t fltOptimizeNodes(SFilterInfo *pInfo, SNode** pNode, SFltTreeStat *pStat) { - + //TODO + return TSDB_CODE_SUCCESS; } -int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { +int32_t fltGetDataFromColId(void *param, int32_t id, void **data) { int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; for (int32_t j = 0; j < numOfCols; ++j) { SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, j); if (id == pColInfo->info.colId) { - *data = pColInfo->pData; + *data = pColInfo; break; } } @@ -3589,13 +3598,28 @@ int32_t filterGetDataFromColId(void *param, int32_t id, void **data) { return TSDB_CODE_SUCCESS; } +int32_t fltGetDataFromSlotId(void *param, int32_t id, void **data) { + int32_t numOfCols = ((SFilterColumnParam *)param)->numOfCols; + SArray* pDataBlock = ((SFilterColumnParam *)param)->pDataBlock; + if (id < 0 || id >= numOfCols || id >= taosArrayGetSize(pDataBlock)) { + fltError("invalid slot id, id:%d, numOfCols:%d, arraySize:%d", id, numOfCols, (int32_t)taosArrayGetSize(pDataBlock)); + return TSDB_CODE_QRY_APP_ERROR; + } + + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, id); + *data = pColInfo; + + return TSDB_CODE_SUCCESS; +} + -int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, false); + +int32_t filterSetDataFromSlotId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromSlotId, false); } -int32_t filterSetDataFromColId(SFilterInfo *info, void *param, filer_get_col_from_id fp) { - return fltSetColFieldDataImpl(info, param, fp, true); +int32_t filterSetDataFromColId(SFilterInfo *info, void *param) { + return fltSetColFieldDataImpl(info, param, fltGetDataFromColId, true); } @@ -3623,6 +3647,8 @@ int32_t filterInitFromNode(SNode* pNode, SFilterInfo **pInfo, uint32_t options) SFltTreeStat stat = {0}; FLT_ERR_JRET(fltReviseNodes(info, &pNode, &stat)); + info->scalarMode = stat.scalarMode; + if (!info->scalarMode) { FLT_ERR_JRET(fltInitFromNode(pNode, info, options)); } else { @@ -3644,10 +3670,26 @@ _return: bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnDataAgg *statis, int16_t numOfCols) { if (info->scalarMode) { SScalarParam output = {0}; - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pSrc, &output)); + SArray *pList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(pList, &pSrc); + + FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = output.data; - return TSDB_CODE_SUCCESS; + taosArrayDestroy(pList); + + *p = output.orig.data; + output.orig.data = NULL; + + sclFreeParam(&output); + + int8_t *r = output.data; + for (int32_t i = 0; i < output.num; ++i) { + if (0 == *(r+i)) { + return false; + } + } + + return true; } return (*info->func)(info, pSrc->info.rows, p, statis, numOfCols); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 336ef8f4f842961b2b06286f8bc041f530ee0f8c..0245226dfcdfdbd360ab1a6e18d20d95839abe1d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1,10 +1,11 @@ #include "nodes.h" -#include "common.h" +#include "tcommon.h" #include "querynodes.h" #include "function.h" #include "functionMgt.h" #include "sclvector.h" #include "sclInt.h" +#include "tep.h" int32_t scalarGetOperatorParamNum(EOperatorType type) { if (OP_TYPE_IS_NULL == type || OP_TYPE_IS_NOT_NULL == type || OP_TYPE_IS_TRUE == type || OP_TYPE_IS_NOT_TRUE == type @@ -87,6 +88,26 @@ _return: SCL_RET(code); } +FORCE_INLINE bool sclIsNull(SScalarParam* param, int32_t idx) { + if (param->dataInBlock) { + return colDataIsNull(param->orig.columnData, 0, idx, NULL); + } + + return param->bitmap ? colDataIsNull_f(param->bitmap, idx) : false; +} + +FORCE_INLINE void sclSetNull(SScalarParam* param, int32_t idx) { + if (NULL == param->bitmap) { + param->bitmap = calloc(BitmapLen(param->num), sizeof(char)); + if (NULL == param->bitmap) { + sclError("calloc %d failed", param->num); + return; + } + } + + colDataSetNull_f(param->bitmap, idx); +} + void sclFreeRes(SHashObj *res) { SScalarParam *p = NULL; @@ -95,7 +116,7 @@ void sclFreeRes(SHashObj *res) { p = (SScalarParam *)pIter; if (p) { - tfree(p->data); + sclFreeParam(p); } pIter = taosHashIterate(res, pIter); @@ -104,19 +125,54 @@ void sclFreeRes(SHashObj *res) { taosHashCleanup(res); } +void sclFreeParamNoData(SScalarParam *param) { + tfree(param->bitmap); +} + + void sclFreeParam(SScalarParam *param) { - tfree(param->data); + sclFreeParamNoData(param); + + if (!param->dataInBlock) { + if (SCL_DATA_TYPE_DUMMY_HASH == param->type) { + taosHashCleanup((SHashObj *)param->orig.data); + } else { + tfree(param->orig.data); + } + } +} + + +int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { + if (TSDB_DATA_TYPE_NULL == pNode->node.resType.type) { + return TSDB_CODE_SUCCESS; + } + + *res = malloc(pNode->node.resType.bytes); + if (NULL == (*res)) { + sclError("malloc %d failed", pNode->node.resType.bytes); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + memcpy(*res, nodesGetValueFromNode(pNode), pNode->node.resType.bytes); + + return TSDB_CODE_SUCCESS; } int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; + //SCL_ERR_RET(sclCopyValueNodeValue(valueNode, ¶m->data)); param->data = nodesGetValueFromNode(valueNode); + param->orig.data = param->data; param->num = 1; param->type = valueNode->node.resType.type; param->bytes = valueNode->node.resType.bytes; - param->colData = false; + if (TSDB_DATA_TYPE_NULL == param->type) { + sclSetNull(param, 0); + } + param->dataInBlock = false; break; } @@ -128,34 +184,44 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SCL_ERR_RET(scalarGenerateSetFromList(¶m->data, node, nodeList->dataType.type)); + param->orig.data = param->data; param->num = 1; param->type = SCL_DATA_TYPE_DUMMY_HASH; - param->colData = false; - + param->dataInBlock = false; + + if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { + taosHashCleanup(param->orig.data); + param->orig.data = NULL; + sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } break; } - case QUERY_NODE_COLUMN_REF: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); + case QUERY_NODE_COLUMN: { + if (NULL == ctx->pBlockList) { + sclError("invalid node type for constant calculating, type:%d, src:%p", nodeType(node), ctx->pBlockList); SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SColumnRefNode *ref = (SColumnRefNode *)node; - if (ref->slotId >= taosArrayGetSize(ctx->pSrc->pDataBlock)) { - sclError("column ref slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(ctx->pSrc->pDataBlock)); + SColumnNode *ref = (SColumnNode *)node; + if (ref->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("column tupleId is too big, tupleId:%d, dataBlockNum:%d", ref->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(ctx->pSrc->pDataBlock, ref->slotId); - if (IS_VAR_DATA_TYPE(columnData->info.type)) { - param->data = columnData; - param->colData = true; - } else { - param->data = columnData->pData; - param->colData = false; + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, ref->dataBlockId); + + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + + SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId); + param->data = NULL; + param->orig.columnData = columnData; + param->dataInBlock = true; - param->num = ctx->pSrc->info.rows; + param->num = block->info.rows; param->type = columnData->info.type; param->bytes = columnData->info.bytes; @@ -163,11 +229,6 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_OPERATOR: { - if (NULL == ctx) { - sclError("invalid node type for constant calculating, type:%d, ctx:%p", nodeType(node), ctx); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); - } - SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, &node, POINTER_BYTES); if (NULL == res) { sclError("no result for node, type:%d, node:%p", nodeType(node), node); @@ -192,20 +253,26 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclParamMoveNext(SScalarParam *params, int32_t num) { +int32_t sclMoveParamListData(SScalarParam *params, int32_t listNum, int32_t idx) { SScalarParam *param = NULL; - for (int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < listNum; ++i) { param = params + i; if (1 == param->num) { continue; } - if (IS_VAR_DATA_TYPE(param->type)) { - param->data = (char *)(param->data) + varDataTLen(param->data); + if (param->dataInBlock) { + param->data = colDataGetData(param->orig.columnData, idx); + } else if (idx) { + if (IS_VAR_DATA_TYPE(param->type)) { + param->data = (char *)(param->data) + varDataTLen(param->data); + } else { + param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + } } else { - param->data = (char *)(param->data) + tDataTypes[param->type].bytes; + param->data = param->orig.data; } } @@ -281,7 +348,7 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu SScalarFuncExecFuncs ffpSet = {0}; int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); if (code) { - sclError( "fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_RET(code); } @@ -295,23 +362,27 @@ int32_t sclExecFuncion(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outpu sclError("calloc %d failed", (int32_t)(rowNum * sizeof(tDataTypes[output->type].bytes))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + code = (*ffpSet.process)(params, node->pParameterList->length, output); if (code) { - sclError( "scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); SCL_ERR_JRET(code); } - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } @@ -346,12 +417,14 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; - void *data = output->data; - bool value = false; for (int32_t i = 0; i < rowNum; ++i) { + sclMoveParamListData(output, 1, i); + sclMoveParamListData(params, node->pParameterList->length, i); + for (int32_t m = 0; m < node->pParameterList->length; ++m) { GET_TYPED_DATA(value, bool, params[m].type, params[m].data); @@ -365,17 +438,14 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } *(bool *)output->data = value; - - sclParamMoveNext(output, 1); - sclParamMoveNext(params, node->pParameterList->length); } - output->data = data; - - return TSDB_CODE_SUCCESS; - _return: + for (int32_t i = 0; i < node->pParameterList->length; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); SCL_RET(code); } @@ -395,6 +465,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp sclError("calloc %d failed", (int32_t)rowNum * tDataTypes[output->type].bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + output->orig.data = output->data; _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); @@ -402,23 +473,27 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp SScalarParam* pLeft = ¶ms[0]; SScalarParam* pRight = paramNum > 1 ? ¶ms[1] : NULL; - OperatorFn(pLeft, pRight, output->data, TSDB_ORDER_ASC); - - return TSDB_CODE_SUCCESS; + OperatorFn(pLeft, pRight, output, TSDB_ORDER_ASC); _return: + + for (int32_t i = 0; i < paramNum; ++i) { + sclFreeParamNoData(params + i); + } + tfree(params); + SCL_RET(code); } -EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { +EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecFuncion(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecFuncion(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -426,7 +501,7 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -447,12 +522,12 @@ EDealRes sclRewriteFunction(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { +EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecLogic(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecLogic(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -460,7 +535,7 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -481,12 +556,12 @@ EDealRes sclRewriteLogic(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { +EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; SScalarParam output = {0}; - *(int32_t *)pContext = sclExecOperator(node, NULL, &output); - if (*(int32_t *)pContext) { + ctx->code = sclExecOperator(node, ctx, &output); + if (ctx->code) { return DEAL_RES_ERROR; } @@ -494,7 +569,7 @@ EDealRes sclRewriteOperator(SNode** pNode, void* pContext) { if (NULL == res) { sclError("make value node failed"); sclFreeParam(&output); - *(int32_t *)pContext = TSDB_CODE_QRY_OUT_OF_MEMORY; + ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; } @@ -521,28 +596,29 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } + SScalarCtx *ctx = (SScalarCtx *)pContext; + if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { - return sclRewriteFunction(pNode, pContext); + return sclRewriteFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pNode)) { - return sclRewriteLogic(pNode, pContext); + return sclRewriteLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { - return sclRewriteOperator(pNode, pContext); + return sclRewriteOperator(pNode, ctx); } sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); - - *(int32_t *)pContext = TSDB_CODE_QRY_INVALID_INPUT; + + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; return DEAL_RES_ERROR; } -EDealRes sclWalkFunction(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)pNode; SScalarParam output = {0}; @@ -560,8 +636,7 @@ EDealRes sclWalkFunction(SNode* pNode, void* pContext) { } -EDealRes sclWalkLogic(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkLogic(SNode* pNode, SScalarCtx *ctx) { SLogicConditionNode *node = (SLogicConditionNode *)pNode; SScalarParam output = {0}; @@ -579,8 +654,7 @@ EDealRes sclWalkLogic(SNode* pNode, void* pContext) { } -EDealRes sclWalkOperator(SNode* pNode, void* pContext) { - SScalarCtx *ctx = (SScalarCtx *)pContext; +EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)pNode; SScalarParam output = {0}; @@ -597,27 +671,69 @@ EDealRes sclWalkOperator(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } +EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { + STargetNode *target = (STargetNode *)pNode; + + if (target->dataBlockId >= taosArrayGetSize(ctx->pBlockList)) { + sclError("target tupleId is too big, tupleId:%d, dataBlockNum:%d", target->dataBlockId, (int32_t)taosArrayGetSize(ctx->pBlockList)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, target->dataBlockId); + if (target->slotId >= taosArrayGetSize(block->pDataBlock)) { + sclError("target slot not exist, dataBlockId:%d, slotId:%d, dataBlockNum:%d", target->dataBlockId, target->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); + ctx->code = TSDB_CODE_QRY_INVALID_INPUT; + return DEAL_RES_ERROR; + } + + SColumnInfoData *col = taosArrayGet(block->pDataBlock, target->slotId); + + SScalarParam *res = (SScalarParam *)taosHashGet(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", target->pExpr, nodeType(target->pExpr)); + ctx->code = TSDB_CODE_QRY_APP_ERROR; + return DEAL_RES_ERROR; + } + + for (int32_t i = 0; i < res->num; ++i) { + sclMoveParamListData(res, 1, i); + + colDataAppend(col, i, res->data, sclIsNull(res, i)); + } + + sclFreeParam(res); + + taosHashRemove(ctx->pRes, (void *)&target->pExpr, POINTER_BYTES); + + return DEAL_RES_CONTINUE; +} + EDealRes sclCalcWalker(SNode* pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN_REF == nodeType(pNode)) { + if (QUERY_NODE_VALUE == nodeType(pNode) || QUERY_NODE_NODE_LIST == nodeType(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { return DEAL_RES_CONTINUE; } + + SScalarCtx *ctx = (SScalarCtx *)pContext; if (QUERY_NODE_FUNCTION == nodeType(pNode)) { - return sclWalkFunction(pNode, pContext); + return sclWalkFunction(pNode, ctx); } if (QUERY_NODE_LOGIC_CONDITION == nodeType(pNode)) { - return sclWalkLogic(pNode, pContext); + return sclWalkLogic(pNode, ctx); } if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - return sclWalkOperator(pNode, pContext); + return sclWalkOperator(pNode, ctx); } - sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); + if (QUERY_NODE_TARGET == nodeType(pNode)) { + return sclWalkTarget(pNode, ctx); + } - SScalarCtx *ctx = (SScalarCtx *)pContext; + sclError("invalid node type for scalar calculating, type:%d", nodeType(pNode)); ctx->code = TSDB_CODE_QRY_INVALID_INPUT; @@ -632,26 +748,33 @@ int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { } int32_t code = 0; + SScalarCtx ctx = {0}; + ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == ctx.pRes) { + sclError("taosHashInit failed, num:%d", SCL_DEFAULT_OP_NUM); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } - nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&code); + nodesRewriteNodePostOrder(&pNode, sclConstantsRewriter, (void *)&ctx); - if (code) { - nodesDestroyNode(pNode); - SCL_ERR_RET(code); - } + SCL_ERR_JRET(ctx.code); *pRes = pNode; - SCL_RET(code); +_return: + + sclFreeRes(ctx.pRes); + + return code; } -int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { - if (NULL == pNode || NULL == pSrc || NULL == pDst) { +int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { + if (NULL == pNode || NULL == pBlockList) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } int32_t code = 0; - SScalarCtx ctx = {.code = 0, .pSrc = pSrc}; + SScalarCtx ctx = {.code = 0, .pBlockList = pBlockList}; ctx.pRes = taosHashInit(SCL_DEFAULT_OP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == ctx.pRes) { @@ -661,23 +784,28 @@ int32_t scalarCalculate(SNode *pNode, SSDataBlock *pSrc, SScalarParam *pDst) { nodesWalkNodePostOrder(pNode, sclCalcWalker, (void *)&ctx); - if (ctx.code) { - nodesDestroyNode(pNode); - sclFreeRes(ctx.pRes); - SCL_ERR_RET(ctx.code); - } + SCL_ERR_JRET(ctx.code); - SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); - if (NULL == res) { - sclError("no res for calculating, node:%p, type:%d", pNode, nodeType(pNode)); - SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); + if (pDst) { + SScalarParam *res = (SScalarParam *)taosHashGet(ctx.pRes, (void *)&pNode, POINTER_BYTES); + if (NULL == res) { + sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); + SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + sclMoveParamListData(res, 1, 0); + + *pDst = *res; + + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } - - *pDst = *res; - nodesDestroyNode(pNode); +_return: + + //nodesDestroyNode(pNode); + sclFreeRes(ctx.pRes); - return TSDB_CODE_SUCCESS; + return code; } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 85af663313eeb64669d4066274202308a320297c..3c431ff33fde08326adf1797a183452197bc0505 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -226,7 +226,7 @@ void* getVectorValueAddr_default(void *src, int32_t index) { return src; } void* getVectorValueAddr_VAR(void *src, int32_t index) { - return colDataGet((SColumnInfoData *)src, index); + return colDataGetData((SColumnInfoData *)src, index); } _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { @@ -261,15 +261,89 @@ _getValueAddr_fn_t getVectorValueAddrFn(int32_t srcType) { return p; } +static FORCE_INLINE void varToSigned(char *buf, SScalarParam* pOut, int32_t outType) { + int64_t value = strtoll(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void varToUnsigned(char *buf, SScalarParam* pOut, int32_t outType) { + uint64_t value = strtoull(buf, NULL, 10); + SET_TYPED_DATA(pOut->data, outType, value); +} + +static FORCE_INLINE void varToFloat(char *buf, SScalarParam* pOut, int32_t outType) { + double value = strtod(buf, NULL); + SET_TYPED_DATA(pOut->data, outType, value); +} + + +int32_t vectorConvertFromVarData(SScalarParam* pIn, SScalarParam* pOut, int32_t inType, int32_t outType) { + int32_t bufSize = 0; + char *tmp = NULL; + _bufConverteFunc func = NULL; + + if (IS_SIGNED_NUMERIC_TYPE(outType) || TSDB_DATA_TYPE_TIMESTAMP == outType || TSDB_DATA_TYPE_BOOL == outType) { + func = varToSigned; + } else if (IS_UNSIGNED_NUMERIC_TYPE(outType)) { + func = varToUnsigned; + } else if (IS_FLOAT_TYPE(outType)) { + func = varToFloat; + } else { + sclError("invalid convert outType:%d", outType); + return TSDB_CODE_QRY_APP_ERROR; + } + + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; + } + + if (TSDB_DATA_TYPE_BINARY == inType) { + if (varDataLen(pIn->data) >= bufSize) { + bufSize = varDataLen(pIn->data) + 1; + tmp = realloc(tmp, bufSize); + } + + memcpy(tmp, varDataVal(pIn->data), varDataLen(pIn->data)); + tmp[varDataLen(pIn->data)] = 0; + } else { + if (varDataLen(pIn->data) * TSDB_NCHAR_SIZE >= bufSize) { + bufSize = varDataLen(pIn->data) * TSDB_NCHAR_SIZE + 1; + tmp = realloc(tmp, bufSize); + } + + int len = taosUcs4ToMbs(varDataVal(pIn->data), varDataLen(pIn->data), tmp); + if (len < 0){ + sclError("castConvert taosUcs4ToMbs error 1"); + tfree(tmp); + return TSDB_CODE_QRY_APP_ERROR; + } + + tmp[len] = 0; + } + + (*func)(tmp, pOut, outType); + } + + tfree(tmp); + + return TSDB_CODE_SUCCESS; +} int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { int16_t inType = pIn->type; int16_t inBytes = pIn->bytes; - char *input = pIn->data; int16_t outType = pOut->type; int16_t outBytes = pOut->bytes; - char *output = pOut->data; + if (inType == TSDB_DATA_TYPE_BINARY || inType == TSDB_DATA_TYPE_NCHAR) { + return vectorConvertFromVarData(pIn, pOut, inType, outType); + } + switch (outType) { case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: @@ -277,239 +351,55 @@ int32_t vectorConvertImpl(SScalarParam* pIn, SScalarParam* pOut) { case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - int64_t value = strtoll(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } + + int64_t value = 0; - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + GET_TYPED_DATA(value, int64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: case TSDB_DATA_TYPE_UBIGINT: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - uint64_t value = strtoull(tmp, NULL, 10); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - uint64_t value = 0; - GET_TYPED_DATA(value, uint64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + uint64_t value = 0; + + GET_TYPED_DATA(value, uint64_t, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: - if (inType == TSDB_DATA_TYPE_BINARY) { - int32_t bufSize = varDataLen(input) + 1; - char *tmp = malloc(bufSize); - if (NULL == tmp) { - sclError("malloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } + for (int32_t i = 0; i < pIn->num; ++i) { + sclMoveParamListData(pIn, 1, i); + sclMoveParamListData(pOut, 1, i); - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input) >= bufSize) { - bufSize = varDataLen(input) + 1; - tmp = realloc(tmp, bufSize); - } - - memcpy(tmp, varDataVal(input), varDataLen(input)); - tmp[varDataLen(input)] = 0; - - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; + if (sclIsNull(pIn, i)) { + sclSetNull(pOut, i); + continue; } - tfree(tmp); - } else if (inType == TSDB_DATA_TYPE_NCHAR) { - int32_t bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - char *tmp = calloc(1, bufSize); - if (NULL == tmp) { - sclError("calloc %d failed", bufSize); - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - if (varDataLen(input)* TSDB_NCHAR_SIZE >= bufSize) { - bufSize = varDataLen(input) * TSDB_NCHAR_SIZE + 1; - tmp = realloc(tmp, bufSize); - } - - int len = taosUcs4ToMbs(varDataVal(input), varDataLen(input), tmp); - if (len < 0){ - sclError("castConvert taosUcs4ToMbs error 1"); - tfree(tmp); - return TSDB_CODE_QRY_APP_ERROR; - } - - tmp[len] = 0; - double value = strtod(tmp, NULL); - SET_TYPED_DATA(output, outType, value); - } - - input += varDataLen(input) + VARSTR_HEADER_SIZE; - output += tDataTypes[outType].bytes; - } - - tfree(tmp); - } else { - for (int32_t i = 0; i < pIn->num; ++i) { - if (isNull(input, inType)) { - assignVal(output, getNullValue(outType), 0, outType); - } else { - int64_t value = 0; - GET_TYPED_DATA(value, int64_t, inType, input); - SET_TYPED_DATA(output, outType, value); - } - - input += tDataTypes[inType].bytes; - output += tDataTypes[outType].bytes; - } + double value = 0; + + GET_TYPED_DATA(value, double, inType, pIn->data); + SET_TYPED_DATA(pOut->data, outType, value); } break; default: @@ -595,6 +485,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p if (NULL == paramOut1->data) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut1->orig.data = paramOut1->data; code = vectorConvertImpl(param1, paramOut1); if (code) { @@ -612,6 +503,7 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p tfree(paramOut1->data); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + paramOut2->orig.data = paramOut2->data; code = vectorConvertImpl(param2, paramOut2); if (code) { @@ -624,24 +516,21 @@ int32_t vectorConvert(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam* p return TSDB_CODE_SUCCESS; } -void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorMath(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, _mathFunc func) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; - - SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; - SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; + double leftv = 0, rightv = 0; + bool isNull = false; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(double)); if (NULL == leftParam.data) { sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -651,64 +540,237 @@ void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } + rightParam.orig.data = rightParam.data; - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; + if (vectorConvertImpl(pRight, &rightParam)) { + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); + return; } + pRight = &rightParam; + } + + if (pLeft->num == pRight->num) { + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } else if (pLeft->num == 1) { + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } else if (pRight->num == 1) { + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); + continue; + } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, (*func)(leftv, rightv, &isNull)); + if (isNull) { + sclSetNull(pOut, i); + isNull = false; + } + } + } + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); +} + +double mathAdd(double leftv, double rightv, bool *isNull) { + return leftv + rightv; +} + +double mathSub(double leftv, double rightv, bool *isNull) { + return leftv - rightv; +} + +double mathMultiply(double leftv, double rightv, bool *isNull) { + return leftv * rightv; +} + +double mathDivide(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv / rightv; +} + +double mathRemainder(double leftv, double rightv, bool *isNull) { + double zero = 0; + if (0 == compareDoubleVal(&rightv, &zero)) { + *isNull = true; + return zero; + } + + return leftv - ((int64_t)(leftv / rightv)) * rightv; +} + + +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathAdd); +} + +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathSub); +} + +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathMultiply); +} + +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathDivide); +} + +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorMath(pLeft, pRight, pOut, _ord, mathRemainder); +} + +#if 0 +void vectorAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; + int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; + SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num, .dataInBlock = false}; + SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num, .dataInBlock = false}; + if (IS_VAR_DATA_TYPE(pLeft->type)) { + leftParam.data = calloc(leftParam.num, sizeof(double)); + if (NULL == leftParam.data) { + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); + return; + } + leftParam.orig.data = leftParam.data; + + if (vectorConvertImpl(pLeft, &leftParam)) { + return; + } + pLeft = &leftParam; + } + if (IS_VAR_DATA_TYPE(pRight->type)) { + rightParam.data = calloc(rightParam.num, sizeof(double)); + if (NULL == rightParam.data) { + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); + return; + } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); - _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) + getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) + getVectorDoubleValueFnRight(pRight->data,i)); + } + + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) + getVectorDoubleValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + SET_DOUBLE_VAL(pOut->data, leftv + rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -718,12 +780,8 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -733,62 +791,75 @@ void vectorSub(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _or rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); - continue; + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); + continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); - } + + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) - getVectorDoubleValueFnRight(pRight->data, i)); + } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) - getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) - getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -798,12 +869,8 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -813,64 +880,76 @@ void vectorMultiply(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_ rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) * getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); + } + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) * getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); + + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) * getVectorDoubleValueFnRight(pRight->data,0)); } - } + } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -880,12 +959,8 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -895,71 +970,73 @@ void vectorDivide(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double *output=(double*)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, getVectorDoubleValueFnLeft(pLeft->data, i) / getVectorDoubleValueFnRight(pRight->data, i)); } } else if (pLeft->num == 1) { - double left = getVectorDoubleValueFnLeft(pLeft->data, 0); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_DOUBLE_NULL(output); + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; - } + } - SET_DOUBLE_VAL(output,left / getVectorDoubleValueFnRight(pRight->data,i)); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data, 0) / getVectorDoubleValueFnRight(pRight->data,i)); } } else if (pRight->num == 1) { - double right = getVectorDoubleValueFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_DOUBLE_VAL(output, getVectorDoubleValueFnLeft(pLeft->data, i) / right); + SET_DOUBLE_VAL(pOut->data,getVectorDoubleValueFnLeft(pLeft->data,i) / getVectorDoubleValueFnRight(pRight->data,0)); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + double leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_DOUBLE, .num = pRight->num}; @@ -969,12 +1046,8 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -984,94 +1057,98 @@ void vectorRemainder(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32 rightParam.data = calloc(rightParam.num, sizeof(double)); if (NULL == rightParam.data) { sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); - tfree(leftParam.data); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - double * output = (double *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getDoubleValue_fn_t getVectorDoubleValueFnLeft = getVectorDoubleValueFn(pLeft->type); _getDoubleValue_fn_t getVectorDoubleValueFnRight = getVectorDoubleValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } - double left = getVectorDoubleValueFnLeft(pLeft->data, i); - double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + GET_TYPED_DATA(leftv, double, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, double, pRight->type, pRight->data); + + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pLeft->num == 1) { double left = getVectorDoubleValueFnLeft(pLeft->data, 0); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(&left, pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, i)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double right = getVectorDoubleValueFnRight(pRight->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } else if (pRight->num == 1) { double right = getVectorDoubleValueFnRight(pRight->data, 0); - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(&right, pRight->type)) { - SET_DOUBLE_NULL(output); + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } double v, u = 0.0; - GET_TYPED_DATA(v, double, pRight->type, getVectorValueAddrFnRight(pRight->data, 0)); + GET_TYPED_DATA(v, double, pRight->type, pRight->data); if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &u) == 0) { - SET_DOUBLE_NULL(output); + sclSetNull(pOut, i); continue; } double left = getVectorDoubleValueFnLeft(pLeft->data, i); - SET_DOUBLE_VAL(output, left - ((int64_t)(left / right)) * right); + SET_DOUBLE_VAL(pOut->data, left - ((int64_t)(left / right)) * right); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } +#endif + void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { int32_t len = pLeft->bytes + pRight->bytes; @@ -1119,28 +1196,24 @@ void vectorConcat(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t varDataSetLen(output, varDataLen(left) + varDataLen(pRight->data)); } } - } -void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1149,80 +1222,95 @@ void vectorBitAnd(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); + _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) & getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) & getVectorBigintValueFnRight(pRight->data,i)); + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) & getVectorBigintValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv & rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; + int64_t leftv = 0, rightv = 0; SScalarParam leftParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pLeft->num}; SScalarParam rightParam = {.type = TSDB_DATA_TYPE_BIGINT, .num = pRight->num}; if (IS_VAR_DATA_TYPE(pLeft->type)) { leftParam.data = calloc(leftParam.num, sizeof(int64_t)); if (NULL == leftParam.data) { - sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(int64_t))); + sclError("malloc %d failed", (int32_t)(leftParam.num * sizeof(double))); return; } + leftParam.orig.data = leftParam.data; - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - if (vectorConvertImpl(pLeft, &leftParam)) { return; } @@ -1231,131 +1319,139 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ if (IS_VAR_DATA_TYPE(pRight->type)) { rightParam.data = calloc(rightParam.num, sizeof(int64_t)); if (NULL == rightParam.data) { - sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(int64_t))); - tfree(leftParam.data); + sclError("malloc %d failed", (int32_t)(rightParam.num * sizeof(double))); + sclFreeParam(&leftParam); return; } - - if (pRight->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pRight->data; - pRight->data = colInfo->pData; - } + rightParam.orig.data = rightParam.data; if (vectorConvertImpl(pRight, &rightParam)) { - tfree(leftParam.data); - tfree(rightParam.data); + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); return; } pRight = &rightParam; } - int64_t *output=(int64_t *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - _getValueAddr_fn_t getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); _getBigintValue_fn_t getVectorBigintValueFnLeft = getVectorBigintValueFn(pLeft->type); _getBigintValue_fn_t getVectorBigintValueFnRight = getVectorBigintValueFn(pRight->type); if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - SET_BIGINT_NULL(output); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output, getVectorBigintValueFnLeft(pLeft->data, i) | getVectorBigintValueFnRight(pRight->data, i)); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pLeft->num == 1) { - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, 0), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pLeft, 1, 0); + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data, 0) | getVectorBigintValueFnRight(pRight->data,i)); + + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } else if (pRight->num == 1) { - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,0), pRight->type)) { - SET_BIGINT_NULL(output); + sclMoveParamListData(pRight, 1, 0); + GET_TYPED_DATA(rightv, int64_t, pRight->type, pRight->data); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - SET_BIGINT_VAL(output,getVectorBigintValueFnLeft(pLeft->data,i) | getVectorBigintValueFnRight(pRight->data,0)); + + GET_TYPED_DATA(leftv, int64_t, pLeft->type, pLeft->data); + + SET_BIGINT_VAL(pOut->data, leftv | rightv); } } - tfree(leftParam.data); - tfree(rightParam.data); + + sclFreeParam(&leftParam); + sclFreeParam(&rightParam); } -void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompareImpl(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; __compar_fn_t fp = filterGetCompFunc(pLeft->type, optr); bool res = false; - - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - _getValueAddr_fn_t getVectorValueAddrFnRight = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - if (IS_VAR_DATA_TYPE(pRight->type) && !pRight->colData) { - getVectorValueAddrFnRight = getVectorValueAddr_default; - } else { - getVectorValueAddrFnRight = getVectorValueAddrFn(pRight->type); - } - if (pLeft->num == pRight->num) { - for (; i < pRight->num && i >= 0; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data, i), pLeft->type) || - isNull(getVectorValueAddrFnRight(pRight->data, i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i < pRight->num && i >= 0; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data, i), getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pLeft->num == 1) { - void *leftData = getVectorValueAddrFnLeft(pLeft->data, 0); + sclMoveParamListData(pLeft, 1, 0); - for (; i >= 0 && i < pRight->num; i += step, output += 1) { - if (isNull(leftData, pLeft->type) || isNull(getVectorValueAddrFnRight(pRight->data,i), pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + for (; i >= 0 && i < pRight->num; i += step) { + sclMoveParamListData(pRight, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, 0) || sclIsNull(pRight, i)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, leftData, getVectorValueAddrFnRight(pRight->data,i)); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); + + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } else if (pRight->num == 1) { - void *rightData = getVectorValueAddrFnRight(pRight->data, 0); - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type) || isNull(rightData, pRight->type)) { - res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + sclMoveParamListData(pRight, 1, 0); + + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i) || sclIsNull(pRight, 0)) { + sclSetNull(pOut, i); continue; } - res = filterDoCompare(fp, optr, getVectorValueAddrFnLeft(pLeft->data,i), rightData); + res = filterDoCompare(fp, optr, pLeft->data, pRight->data); - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } } -void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord, int32_t optr) { +void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord, int32_t optr) { SScalarParam pLeftOut = {0}; SScalarParam pRightOut = {0}; @@ -1377,118 +1473,103 @@ void vectorCompare(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t param2 = pRight; } - vectorCompareImpl(param1, param2, out, _ord, optr); + vectorCompareImpl(param1, param2, pOut, _ord, optr); + + sclFreeParam(&pLeftOut); + sclFreeParam(&pRightOut); } -void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_THAN); +void vectorGreater(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_THAN); } -void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_GREATER_EQUAL); +void vectorGreaterEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_GREATER_EQUAL); } -void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_THAN); +void vectorLower(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_THAN); } -void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LOWER_EQUAL); +void vectorLowerEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LOWER_EQUAL); } -void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_EQUAL); +void vectorEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_EQUAL); } -void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_EQUAL); +void vectorNotEqual(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_EQUAL); } -void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_IN); +void vectorIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_IN); } -void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_IN); +void vectorNotIn(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_IN); } -void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_LIKE); +void vectorLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_LIKE); } -void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NOT_LIKE); +void vectorNotLike(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NOT_LIKE); } -void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_MATCH); +void vectorMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_MATCH); } -void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - vectorCompare(pLeft, pRight, out, _ord, OP_TYPE_NMATCH); +void vectorNotMatch(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorCompare(pLeft, pRight, pOut, _ord, OP_TYPE_NMATCH); } -void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorIsNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - bool *output=(bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } } -void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { +void vectorNotNull(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : TMAX(pLeft->num, pRight->num) - 1; int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; bool res = false; - - bool *output = (bool *)out; - _getValueAddr_fn_t getVectorValueAddrFnLeft = NULL; - if (IS_VAR_DATA_TYPE(pLeft->type) && !pLeft->colData) { - getVectorValueAddrFnLeft = getVectorValueAddr_default; - } else { - getVectorValueAddrFnLeft = getVectorValueAddrFn(pLeft->type); - } - - for (; i >= 0 && i < pLeft->num; i += step, output += 1) { - if (isNull(getVectorValueAddrFnLeft(pLeft->data,i), pLeft->type)) { + for (; i >= 0 && i < pLeft->num; i += step) { + sclMoveParamListData(pLeft, 1, i); + sclMoveParamListData(pOut, 1, i); + + if (sclIsNull(pLeft, i)) { res = false; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); continue; } res = true; - SET_TYPED_DATA(output, TSDB_DATA_TYPE_BOOL, res); + SET_TYPED_DATA(pOut->data, TSDB_DATA_TYPE_BOOL, res); } -} -void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, void *out, int32_t _ord) { - SScalarParam output = {.data = out, .num = pLeft->num, .type = TSDB_DATA_TYPE_BOOL}; +} - if (pLeft->colData) { - SColumnInfoData *colInfo = (SColumnInfoData *)pLeft->data; - pLeft->data = colInfo->pData; - } - - vectorConvertImpl(pLeft, &output); +void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { + vectorConvertImpl(pLeft, pOut); } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index db9df9ff085df3da73a4f8d9c24151b07f1dac40..fafc1ea42e860a8c410d75c1e58738f3c1e9d79e 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -51,10 +51,9 @@ void flttInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } @@ -77,16 +76,20 @@ void flttMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; - rnode->tupleId = 0; +void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + static uint64_t dbidx = 0; + + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->dataBlockId = 0; + + sprintf(rnode->dbName, "%" PRIu64, dbidx++); if (NULL == block) { rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 3; *pNode = (SNode *)rnode; return; @@ -101,7 +104,7 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -111,18 +114,25 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 3; *block = res; } else { @@ -132,12 +142,26 @@ void flttMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; - idata.pData = (char *)value; + idata.info.colId = 1 + idx; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); + res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -171,6 +195,17 @@ void flttMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi *pNode = (SNode *)onode; } +void flttMakeLogicNodeFromList(SNode **pNode, ELogicConditionType opType, SNodeList *nodeList) { + SNode *node = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *onode = (SLogicConditionNode *)node; + onode->condType = opType; + onode->node.resType.type = TSDB_DATA_TYPE_BOOL; + onode->node.resType.bytes = sizeof(bool); + + onode->pParameterList = nodeList; + + *pNode = (SNode *)onode; +} void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { SNode *node = nodesMakeNode(QUERY_NODE_NODE_LIST); @@ -184,17 +219,36 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { } +TEST(timerangeTest, greater) { + SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; + bool eRes[5] = {false, false, true, true, true}; + SScalarParam res = {0}; + int64_t tsmall = 222, tbig = 333; + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode1, &filter, FLT_OPTION_NO_REWRITE|FLT_OPTION_TIMESTAMP); + ASSERT_EQ(code, 0); + STimeWindow win = {0}; + code = filterGetTimeRange(filter, &win); + ASSERT_EQ(code, 0); + ASSERT_EQ(win.skey, tsmall); + ASSERT_EQ(win.ekey, INT64_MAX); + filterFreeInfo(filter); + nodesDestroyNode(opNode1); +} + TEST(timerangeTest, greater_and_lower) { - flttInitLogFile(); - SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; SScalarParam res = {0}; int64_t tsmall = 222, tbig = 333; - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); - flttMakeColRefNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); + flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tbig); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol, pval); SNode *list[2] = {0}; @@ -211,31 +265,122 @@ TEST(timerangeTest, greater_and_lower) { ASSERT_EQ(code, 0); ASSERT_EQ(win.skey, tsmall); ASSERT_EQ(win.ekey, tbig); + filterFreeInfo(filter); + nodesDestroyNode(logicNode); } -#if 0 + TEST(columnTest, smallint_column_greater_double_value) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int16_t leftv[5]= {1, 2, 3, 4, 5}; double rightv= 2.5; - bool eRes[5] = {false, false, true, true, true}; + int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnNode *)pLeft)->colId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + blockDataDestroy(src); + nodesDestroyNode(opNode); +} + +TEST(columnTest, int_column_greater_smallint_value) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int32_t leftv[5]= {1, 3, 5, 7, 9}; + int16_t rightv= 4; + int8_t eRes[5] = {0, 0, 1, 1, 1}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SColumnDataAgg stat = {0}; + stat.colId = ((SColumnNode *)pLeft)->colId; + stat.max = 10; + stat.min = 5; + stat.numOfNull = 0; + bool keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, true); + + stat.max = 1; + stat.min = -1; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + stat.max = 10; + stat.min = 5; + stat.numOfNull = rowNum; + keep = filterRangeExecute(filter, &stat, 1, rowNum); + ASSERT_EQ(keep, false); + + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; @@ -244,7 +389,7 @@ TEST(columnTest, int_column_in_double_list) { SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -254,17 +399,35 @@ TEST(columnTest, int_column_in_double_list) { nodesListAppend(list, pRight); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - - int32_t code = scalarCalculate(opNode, src, &res); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + + TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; @@ -292,7 +455,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -303,16 +466,32 @@ TEST(columnTest, binary_column_in_binary_list) { flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; @@ -329,21 +508,36 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } @@ -352,30 +546,46 @@ TEST(columnTest, binary_column_is_null) { char leftv[5][5]= {0}; SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(columnTest, binary_column_is_not_null) { @@ -385,65 +595,126 @@ TEST(columnTest, binary_column_is_not_null) { SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); - for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); - } -} -TEST(logicTest, and_or_and) { + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } -TEST(logicTest, or_and_or) { - -} TEST(opTest, smallint_column_greater_int_column) { + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; + int16_t leftv[5] = {1, -6, -2, 11, 101}; + int32_t rightv[5]= {0, -5, -4, 23, 100}; + bool eRes[5] = {true, false, true, false, true}; + SSDataBlock *src = NULL; + SScalarParam res = {0}; + int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); + flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + TEST(opTest, smallint_value_add_int_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int32_t leftv = 1; - int16_t rightv[5]= {0, -5, -4, 23, 100}; - double eRes[5] = {1.0, -4, -3, 24, 101}; + int16_t rightv[5]= {0, -1, -4, -1, 100}; + bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + + TEST(opTest, bigint_column_multi_binary_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int64_t leftv[5]= {1, 2, 3, 4, 5}; @@ -453,22 +724,38 @@ TEST(opTest, bigint_column_multi_binary_column) { rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); } - double eRes[5] = {0, 2, 6, 12, 20}; + bool eRes[5] = {false, true, true, true, true}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_and_binary_column) { @@ -480,68 +767,119 @@ TEST(opTest, smallint_column_and_binary_column) { rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); } - int64_t eRes[5] = {0, 0, 2, 0, 4}; + bool eRes[5] = {false, false, true, false, true}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } TEST(opTest, smallint_column_or_float_column) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; - int16_t leftv[5]= {1, 2, 3, 4, 5}; - float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; - int64_t eRes[5] = {3, 3, 7, 5, 7}; + int16_t leftv[5]= {1, 2, 0, 4, 5}; + float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; + bool eRes[5] = {true, true, false, true, true}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - flttMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + + TEST(opTest, smallint_column_or_double_value) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; - int16_t leftv[5]= {1, 2, 3, 4, 5}; + int16_t leftv[5]= {0, 2, 3, 0, -1}; double rightv= 10.2; - int64_t eRes[5] = {11, 10, 11, 14, 15}; + bool eRes[5] = {true, true, true, true, true}; SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); + flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, true); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); } + TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; @@ -557,20 +895,389 @@ TEST(opTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(opNode, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(opNode); + blockDataDestroy(src); +} + + +TEST(filterModelogicTest, diff_columns_and_or_and) { + flttInitLogFile(); + + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 1, 0, 0, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); +} + +TEST(filterModelogicTest, same_column_and_or_and) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {1, 1, 0, 0, 0, 1, 1, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + list = nodesMakeList(); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); } -#endif + + +TEST(filterModelogicTest, diff_columns_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1= 3, rightv2= 3; + int8_t eRes[8] = {1, 0, 1, 1, 0, 0, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); +} + +TEST(filterModelogicTest, same_column_or_and_or) { + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8]= {1, 2, 3, 4, 5,-1,-2,-3}; + int32_t rightv1= 3, rightv2= 0, rightv3 = 2, rightv4 = -2; + int8_t eRes[8] = {0, 0, 0, 1, 1, 1, 1, 1}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); + flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); + flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); +} + + + +TEST(scalarModelogicTest, diff_columns_or_and_or) { + flttInitLogFile(); + + SNode *pLeft1 = NULL, *pRight1 = NULL, *pLeft2 = NULL, *pRight2 = NULL, *opNode1 = NULL, *opNode2 = NULL; + SNode *logicNode1 = NULL, *logicNode2 = NULL; + double leftv1[8] = {1, 2, 3, 4, 5,-1,-2,-3}, leftv2[8]= {3.0, 4, 2, 9, -3, 3.9, 4.1, 5.2}; + int32_t rightv1[8]= {5, 8, 2, -3, 9,-7,10, 0}, rightv2[8]= {-3, 5, 8, 2, -9, 11, -4, 0}; + int8_t eRes[8] = {0, 1, 1, 0, 0, 1, 0, 0}; + SSDataBlock *src = NULL; + + SNodeList* list = nodesMakeList(); + + int32_t rowNum = sizeof(leftv1)/sizeof(leftv1[0]); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); + + + list = nodesMakeList(); + + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); + flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); + nodesListAppend(list, opNode1); + + flttMakeColumnNode(&pLeft2, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv2); + flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); + flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); + nodesListAppend(list, opNode2); + + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); + + list = nodesMakeList(); + nodesListAppend(list, logicNode1); + nodesListAppend(list, logicNode2); + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); + + + SFilterInfo *filter = NULL; + int32_t code = filterInitFromNode(logicNode1, &filter, 0); + ASSERT_EQ(code, 0); + + SColumnDataAgg stat = {0}; + SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + code = filterSetDataFromSlotId(filter, ¶m); + ASSERT_EQ(code, 0); + + stat.max = 5; + stat.min = 1; + stat.numOfNull = 0; + int8_t *rowRes = NULL; + bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + ASSERT_EQ(keep, false); + + for (int32_t i = 0; i < rowNum; ++i) { + ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); + } + tfree(rowRes); + filterFreeInfo(filter); + nodesDestroyNode(logicNode1); + blockDataDestroy(src); +} + + int main(int argc, char** argv) { srand(time(NULL)); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2f0a60362dd82b9dbb5e6c53490f95770a82681d..b9aef99088bef2fefb98a7447cd77c6945c1ba76 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -44,20 +44,53 @@ namespace { int64_t scltLeftV = 21, scltRightV = 10; double scltLeftVd = 21.0, scltRightVd = 10.0; +void scltFreeDataBlock(void *block) { + blockDataDestroy(*(SSDataBlock **)block); +} + void scltInitLogFile() { const char *defaultLogFileNamePrefix = "taoslog"; const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; + strcpy(tsLogDir, "/var/log/taos"); - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } +void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { + if (newBlock) { + SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); + res->info.numOfCols = 1; + res->info.rows = rows; + res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + taosArrayPush(res->pDataBlock, &idata); + taosArrayPush(pBlockList, &res); + + blockDataEnsureCapacity(res, rows); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + *slotId = 0; + } else { + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); + res->info.numOfCols++; + SColumnInfoData idata = {0}; + idata.info = *colInfo; + + blockDataEnsureColumnCapacity(&idata, rows); + + taosArrayPush(res->pDataBlock, &idata); + + *dataBlockId = taosArrayGetSize(pBlockList) - 1; + *slotId = taosArrayGetSize(res->pDataBlock) - 1; + } +} void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { SNode *node = nodesMakeNode(QUERY_NODE_VALUE); @@ -67,7 +100,7 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { if (IS_VAR_DATA_TYPE(dataType)) { vnode->datum.p = (char *)malloc(varDataTLen(value)); varDataCopy(vnode->datum.p, value); - vnode->node.resType.bytes = varDataLen(value); + vnode->node.resType.bytes = varDataTLen(value); } else { vnode->node.resType.bytes = tDataTypes[dataType].bytes; assignVal((char *)nodesGetValueFromNode(vnode), (const char *)value, 0, dataType); @@ -76,12 +109,12 @@ void scltMakeValueNode(SNode **pNode, int32_t dataType, void *value) { *pNode = (SNode *)vnode; } -void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { - SNode *node = nodesMakeNode(QUERY_NODE_COLUMN_REF); - SColumnRefNode *rnode = (SColumnRefNode *)node; - rnode->dataType.type = dataType; - rnode->dataType.bytes = dataBytes; - rnode->tupleId = 0; +void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, int32_t dataBytes, int32_t rowNum, void *value) { + SNode *node = nodesMakeNode(QUERY_NODE_COLUMN); + SColumnNode *rnode = (SColumnNode *)node; + rnode->node.resType.type = dataType; + rnode->node.resType.bytes = dataBytes; + rnode->dataBlockId = 0; if (NULL == *block) { SSDataBlock *res = (SSDataBlock *)calloc(1, sizeof(SSDataBlock)); @@ -92,7 +125,7 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = TSDB_DATA_TYPE_NULL; idata.info.bytes = 10; - idata.info.colId = 0; + idata.info.colId = i + 1; int32_t size = idata.info.bytes * rowNum; idata.pData = (char *)calloc(1, size); @@ -102,18 +135,25 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55; - idata.pData = (char *)value; - if (IS_VAR_DATA_TYPE(dataType)) { - idata.varmeta.offset = (int32_t *)calloc(rowNum, sizeof(int32_t)); - for (int32_t i = 0; i < rowNum; ++i) { - idata.varmeta.offset[i] = (dataBytes + VARSTR_HEADER_SIZE) * i; + idata.info.colId = 3; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); + taosArrayPush(res->pDataBlock, &idata); + + blockDataEnsureCapacity(res, rowNum); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; } } - taosArrayPush(res->pDataBlock, &idata); rnode->slotId = 2; - rnode->columnId = 55; + rnode->colId = 3; *block = res; } else { @@ -123,12 +163,26 @@ void scltMakeColRefNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SColumnInfoData idata = {{0}}; idata.info.type = dataType; idata.info.bytes = dataBytes; - idata.info.colId = 55 + idx; - idata.pData = (char *)value; + idata.info.colId = 1 + idx; + int32_t size = idata.info.bytes * rowNum; + idata.pData = (char *)calloc(1, size); taosArrayPush(res->pDataBlock, &idata); + res->info.numOfCols++; + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + + blockDataEnsureColumnCapacity(pColumn, rowNum); + + for (int32_t i = 0; i < rowNum; ++i) { + colDataAppend(pColumn, i, (const char *)value, false); + if (IS_VAR_DATA_TYPE(dataType)) { + value = (char *)value + varDataTLen(value); + } else { + value = (char *)value + dataBytes; + } + } rnode->slotId = idx; - rnode->columnId = 55 + idx; + rnode->colId = 1 + idx; } *pNode = (SNode *)rnode; @@ -173,6 +227,17 @@ void scltMakeLogicNode(SNode **pNode, ELogicConditionType opType, SNode **nodeLi *pNode = (SNode *)onode; } +void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNode *snode) { + SNode *node = nodesMakeNode(QUERY_NODE_TARGET); + STargetNode *onode = (STargetNode *)node; + onode->pExpr = snode; + onode->dataBlockId = dataBlockId; + onode->slotId = slotId; + + *pNode = (SNode *)onode; +} + + } @@ -189,6 +254,7 @@ TEST(constantTest, bigint_add_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftV + scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, double_sub_bigint) { @@ -204,6 +270,7 @@ TEST(constantTest, double_sub_bigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_DOUBLE); ASSERT_EQ(v->datum.d, (scltLeftVd - scltRightV)); + nodesDestroyNode(res); } TEST(constantTest, tinyint_and_smallint) { @@ -219,6 +286,7 @@ TEST(constantTest, tinyint_and_smallint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV & (int64_t)scltRightV); + nodesDestroyNode(res); } TEST(constantTest, bigint_or_double) { @@ -234,6 +302,7 @@ TEST(constantTest, bigint_or_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.i, (int64_t)scltLeftV | (int64_t)scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_or_binary) { @@ -252,6 +321,7 @@ TEST(constantTest, int_or_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT); ASSERT_EQ(v->datum.b, scltLeftV | scltRightV); + nodesDestroyNode(res); } @@ -268,6 +338,7 @@ TEST(constantTest, int_greater_double) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, int_greater_equal_binary) { @@ -286,6 +357,7 @@ TEST(constantTest, int_greater_equal_binary) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV > scltRightVd); + nodesDestroyNode(res); } TEST(constantTest, tinyint_lower_ubigint) { @@ -301,11 +373,13 @@ TEST(constantTest, tinyint_lower_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, scltLeftV < scltRightV); + nodesDestroyNode(res); } TEST(constantTest, usmallint_lower_equal_ubigint) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int64_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_USMALLINT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_UBIGINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -317,11 +391,13 @@ TEST(constantTest, usmallint_lower_equal_ubigint) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv <= rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint1) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = 1, rightv = 1; + int32_t leftv = 1; + int16_t rightv = 1; scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); scltMakeOpNode(&opNode, OP_TYPE_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); @@ -333,6 +409,7 @@ TEST(constantTest, int_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_equal_smallint2) { @@ -349,6 +426,7 @@ TEST(constantTest, int_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv == rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint1) { @@ -365,6 +443,7 @@ TEST(constantTest, int_not_equal_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } TEST(constantTest, int_not_equal_smallint2) { @@ -381,6 +460,7 @@ TEST(constantTest, int_not_equal_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, leftv != rightv); + nodesDestroyNode(res); } @@ -408,6 +488,7 @@ TEST(constantTest, int_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_in_smallint2) { @@ -433,6 +514,7 @@ TEST(constantTest, int_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint1) { @@ -456,6 +538,7 @@ TEST(constantTest, int_not_in_smallint1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_not_in_smallint2) { @@ -481,6 +564,7 @@ TEST(constantTest, int_not_in_smallint2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary1) { @@ -501,6 +585,7 @@ TEST(constantTest, binary_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_like_binary2) { @@ -521,6 +606,7 @@ TEST(constantTest, binary_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary1) { @@ -541,6 +627,7 @@ TEST(constantTest, binary_not_like_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_like_binary2) { @@ -561,6 +648,7 @@ TEST(constantTest, binary_not_like_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary1) { @@ -581,6 +669,7 @@ TEST(constantTest, binary_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_match_binary2) { @@ -601,6 +690,7 @@ TEST(constantTest, binary_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary1) { @@ -621,6 +711,7 @@ TEST(constantTest, binary_not_match_binary1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, binary_not_match_binary2) { @@ -641,6 +732,7 @@ TEST(constantTest, binary_not_match_binary2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null1) { @@ -656,12 +748,13 @@ TEST(constantTest, int_is_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_is_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -671,6 +764,7 @@ TEST(constantTest, int_is_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null1) { @@ -686,12 +780,13 @@ TEST(constantTest, int_is_not_null1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_is_not_null2) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL, *res = NULL; - int32_t leftv = TSDB_DATA_INT_NULL, rightv = 1; - scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); + int32_t leftv = 1, rightv = 1; + scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_NULL, &leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, pRight); int32_t code = scalarCalculateConstants(opNode, &res); @@ -701,6 +796,7 @@ TEST(constantTest, int_is_not_null2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true1) { @@ -718,6 +814,7 @@ TEST(constantTest, int_add_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, int_add_int_is_true2) { @@ -735,6 +832,7 @@ TEST(constantTest, int_add_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } @@ -753,6 +851,7 @@ TEST(constantTest, int_greater_int_is_true1) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); } TEST(constantTest, int_greater_int_is_true2) { @@ -770,6 +869,7 @@ TEST(constantTest, int_greater_int_is_true2) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(constantTest, greater_and_lower) { @@ -794,30 +894,45 @@ TEST(constantTest, greater_and_lower) { SValueNode *v = (SValueNode *)res; ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); } TEST(columnTest, smallint_value_add_int_column) { + scltInitLogFile(); + SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; int32_t leftv = 1; int16_t rightv[5]= {0, -5, -4, 23, 100}; double eRes[5] = {1.0, -4, -3, 24, 101}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); scltMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + SArray *blockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, bigint_column_multi_binary_column) { @@ -831,20 +946,32 @@ TEST(columnTest, bigint_column_multi_binary_column) { } double eRes[5] = {0, 2, 6, 12, 20}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); + + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_DOUBLE, .bytes = sizeof(double)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_DOUBLE); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((double *)res.data + i), eRes[i]); + ASSERT_EQ(*((double *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_and_binary_column) { @@ -858,20 +985,31 @@ TEST(columnTest, smallint_column_and_binary_column) { } int64_t eRes[5] = {0, 0, 2, 0, 4}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_float_column) { @@ -880,20 +1018,31 @@ TEST(columnTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 4.1, 5.2, 6.0}; int64_t eRes[5] = {3, 3, 7, 5, 7}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); - scltMakeColRefNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_or_double_value) { @@ -902,20 +1051,31 @@ TEST(columnTest, smallint_column_or_double_value) { double rightv= 10.2; int64_t eRes[5] = {11, 10, 11, 14, 15}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BIGINT, .bytes = sizeof(int64_t)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BIGINT); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BIGINT); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((int64_t *)res.data + i), eRes[i]); + ASSERT_EQ(*((int64_t *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, smallint_column_greater_double_value) { @@ -924,20 +1084,31 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; bool eRes[5] = {false, false, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); scltMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, int_column_in_double_list) { @@ -946,9 +1117,8 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -958,22 +1128,33 @@ TEST(columnTest, int_column_in_double_list) { nodesListAppend(list, pRight); scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_INT); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); + + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, true, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); - int32_t code = scalarCalculate(opNode, src, &res); + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -996,7 +1177,7 @@ TEST(columnTest, binary_column_in_binary_list) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -1007,14 +1188,26 @@ TEST(columnTest, binary_column_in_binary_list) { scltMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); scltMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_like_binary) { @@ -1022,7 +1215,6 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -1033,28 +1225,41 @@ TEST(columnTest, binary_column_like_binary) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); scltMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } + TEST(columnTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -1065,80 +1270,117 @@ TEST(columnTest, binary_column_is_true) { } int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); scltMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; - bool eRes[5] = {false, false, false, false, true}; + bool eRes[5] = {false, false, true, false, true}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 2, NULL, true); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; bool eRes[5] = {true, true, true, true, false}; - for (int32_t i = 0; i < 4; ++i) { + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); } - - setVardataNull(leftv[4], TSDB_DATA_TYPE_BINARY); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - scltMakeColRefNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + scltMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + + SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); + colDataAppend(pcolumn, 4, NULL, true); scltMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - int32_t code = scalarCalculate(opNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&opNode, dataBlockId, slotId, opNode); + + int32_t code = scalarCalculate(opNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(opNode); } TEST(columnTest, greater_and_lower) { @@ -1150,30 +1392,40 @@ TEST(columnTest, greater_and_lower) { int32_t v4[5]= {5, 3, 4, 2, 6}; bool eRes[5] = {false, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; int32_t rowNum = sizeof(v1)/sizeof(v1[0]); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v2); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, v1); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v2); scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); - scltMakeColRefNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int16_t), rowNum, v3); - scltMakeColRefNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int16_t), rowNum, v4); + scltMakeColumnNode(&pcol1, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, v3); + scltMakeColumnNode(&pcol2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, v4); scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pcol1, pcol2); list[0] = opNode1; list[1] = opNode2; scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); - int32_t code = scalarCalculate(logicNode, src, &res); + SArray *blockList = taosArrayInit(1, POINTER_BYTES); + taosArrayPush(blockList, &src); + SColumnInfo colInfo = {.colId = 1, .type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + int16_t dataBlockId = 0, slotId = 0; + scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); + scltMakeTargetNode(&logicNode, dataBlockId, slotId, logicNode); + + int32_t code = scalarCalculate(logicNode, blockList, NULL); ASSERT_EQ(code, 0); - ASSERT_EQ(res.num, rowNum); - ASSERT_EQ(res.type, TSDB_DATA_TYPE_BOOL); - ASSERT_EQ(res.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); + + SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(blockList); + ASSERT_EQ(res->info.rows, rowNum); + SColumnInfoData *column = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + ASSERT_EQ(column->info.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(column->info.bytes, tDataTypes[TSDB_DATA_TYPE_BOOL].bytes); for (int32_t i = 0; i < rowNum; ++i) { - ASSERT_EQ(*((bool *)res.data + i), eRes[i]); + ASSERT_EQ(*((bool *)colDataGetData(column, i)), eRes[i]); } + taosArrayDestroyEx(blockList, scltFreeDataBlock); + nodesDestroyNode(logicNode); } - int main(int argc, char** argv) { srand(time(NULL)); testing::InitGoogleTest(&argc, argv); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 45e125608a15c92488f146d15839490ccd55dcad..f14b95873ac8b1d4929aa993c8b1e39ba8345f65 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -578,7 +578,7 @@ int32_t schProcessOnJobFailureImpl(SSchJob *pJob, int32_t status, int32_t errCod atomic_store_32(&pJob->errCode, errCode); } - if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + if (atomic_load_8(&pJob->userFetch) || pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } @@ -638,7 +638,7 @@ int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - if ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule) { + if (pJob->attr.syncSchedule) { tsem_post(&pJob->rspSem); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6bfff0197c80debdcb8c9bcf920c3742c9f11776..89d365a7e7fe2410a5cfa8e3e7168b28f096540d 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -66,9 +66,7 @@ void schtInitLogFile() { tsAsyncLog = 0; qDebugFlag = 159; - char temp[128] = {0}; - sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); - if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index 37ee5194c81d337b68e92df096c42a4721ec93eb..cb38d7e3637a4cf11dff6ddc5d3e91354ad81587 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -13,4 +13,8 @@ target_include_directories( sync PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/sync/inc/syncAppendEntries.h b/source/libs/sync/inc/syncAppendEntries.h index 9ca0de19c5fbecb6476d281658a4a5e2f5980451..b7c1c051cc3301e40c2024ee8dfee52fd941b075 100644 --- a/source/libs/sync/inc/syncAppendEntries.h +++ b/source/libs/sync/inc/syncAppendEntries.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncAppendEntriesReply.h b/source/libs/sync/inc/syncAppendEntriesReply.h index 8b5cbf1da53c9ff29e925eafe5ae912a9e28d396..22f8eb464fecb97d93ef45798f4d79fe7e4114db 100644 --- a/source/libs/sync/inc/syncAppendEntriesReply.h +++ b/source/libs/sync/inc/syncAppendEntriesReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncElection.h b/source/libs/sync/inc/syncElection.h index 34dfdb3d0952f1e4ce996223393ade44f787f4a3..7e9e637854f29f35650761e4411357099396d9d4 100644 --- a/source/libs/sync/inc/syncElection.h +++ b/source/libs/sync/inc/syncElection.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/include/util/tstep.h b/source/libs/sync/inc/syncEnv.h similarity index 61% rename from include/util/tstep.h rename to source/libs/sync/inc/syncEnv.h index e4ce7f584b368b87c5e0636abed30d9e201b7f83..f1c4327b692dfbf1c9066d10894dc827196047fd 100644 --- a/include/util/tstep.h +++ b/source/libs/sync/inc/syncEnv.h @@ -13,25 +13,35 @@ * along with this program. If not, see . */ -#ifndef _TD_UTIL_STEP_H_ -#define _TD_UTIL_STEP_H_ +#ifndef _TD_LIBS_SYNC_ENV_H +#define _TD_LIBS_SYNC_ENV_H #ifdef __cplusplus extern "C" { #endif -typedef struct SSteps SSteps; -typedef int32_t (*InitFp)(); -typedef void (*CleanupFp)(); -typedef void (*ReportFp)(char *name, char *desc); +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" +#include "trpc.h" -SSteps *taosStepInit(int32_t maxsize, ReportFp fp); -int32_t taosStepExec(SSteps *steps); -void taosStepCleanup(SSteps *steps); -int32_t taosStepAdd(SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp); +typedef struct SSyncEnv { + void *pTimer; + void *pTimerManager; +} SSyncEnv; + +int32_t syncEnvStart(); + +int32_t syncEnvStop(); + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv); + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_STEP_H_*/ +#endif /*_TD_LIBS_SYNC_ENV_H*/ diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h new file mode 100644 index 0000000000000000000000000000000000000000..4b788efd797fc3b722742da9ae4360496dc2e0b8 --- /dev/null +++ b/source/libs/sync/inc/syncIO.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_LIBS_IO_H +#define _TD_LIBS_IO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include "os.h" +#include "syncInt.h" +#include "taosdef.h" +#include "tqueue.h" +#include "trpc.h" + +typedef struct SSyncIO { + void * serverRpc; + void * clientRpc; + STaosQueue *pMsgQ; + STaosQset * pQset; + pthread_t tid; + int8_t isStart; + + SEpSet epSet; + + void *syncTimer; + void *syncTimerManager; + + int32_t (*start)(struct SSyncIO *ths); + int32_t (*stop)(struct SSyncIO *ths); + int32_t (*ping)(struct SSyncIO *ths); + + int32_t (*onMsg)(struct SSyncIO *ths, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); + int32_t (*destroy)(struct SSyncIO *ths); + + void *pSyncNode; + int32_t (*FpOnPing)(struct SSyncNode *ths, SyncPing *pMsg); + +} SSyncIO; + +extern SSyncIO *gSyncIO; + +int32_t syncIOStart(); +int32_t syncIOStop(); +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg); +SSyncIO *syncIOCreate(); + +static int32_t doSyncIOStart(SSyncIO *io); +static int32_t doSyncIOStop(SSyncIO *io); +static int32_t doSyncIOPing(SSyncIO *io); +static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); +static int32_t doSyncIODestroy(SSyncIO *io); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_LIBS_IO_H*/ diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 551ce83122c519190be0b494bc76d2fafe72b7e6..ad8484662aea70ebb262582e9f5b637af19384eb 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -23,7 +23,11 @@ extern "C" { #include #include #include +#include "sync.h" #include "taosdef.h" +#include "tlog.h" + +extern int32_t sDebugFlag; #define sFatal(...) \ { \ @@ -62,16 +66,81 @@ extern "C" { } \ } +struct SRaft; +typedef struct SRaft SRaft; + +struct SyncPing; +typedef struct SyncPing SyncPing; + +struct SyncPingReply; +typedef struct SyncPingReply SyncPingReply; + +struct SyncRequestVote; +typedef struct SyncRequestVote SyncRequestVote; + +struct SyncRequestVoteReply; +typedef struct SyncRequestVoteReply SyncRequestVoteReply; + +struct SyncAppendEntries; +typedef struct SyncAppendEntries SyncAppendEntries; + +struct SyncAppendEntriesReply; +typedef struct SyncAppendEntriesReply SyncAppendEntriesReply; + typedef struct SSyncNode { - char path[TSDB_FILENAME_LEN]; - int8_t replica; - int8_t quorum; - int8_t selfIndex; - uint32_t vgId; - int32_t refCount; - int64_t rid; + int8_t replica; + int8_t quorum; + + SyncGroupId vgId; + SSyncCfg syncCfg; + char path[TSDB_FILENAME_LEN]; + + SRaft* pRaft; + + int32_t (*FpPing)(struct SSyncNode* ths, const SyncPing* pMsg); + + int32_t (*FpOnPing)(struct SSyncNode* ths, SyncPing* pMsg); + + int32_t (*FpOnPingReply)(struct SSyncNode* ths, SyncPingReply* pMsg); + + int32_t (*FpRequestVote)(struct SSyncNode* ths, const SyncRequestVote* pMsg); + + int32_t (*FpOnRequestVote)(struct SSyncNode* ths, SyncRequestVote* pMsg); + + int32_t (*FpOnRequestVoteReply)(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); + + int32_t (*FpAppendEntries)(struct SSyncNode* ths, const SyncAppendEntries* pMsg); + + int32_t (*FpOnAppendEntries)(struct SSyncNode* ths, SyncAppendEntries* pMsg); + + int32_t (*FpOnAppendEntriesReply)(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + + int32_t (*FpSendMsg)(void* handle, const SEpSet* pEpSet, SRpcMsg* pMsg); + } SSyncNode; +SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo); + +void syncNodeClose(SSyncNode* pSyncNode); + +static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg); + +static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg); + +static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg); + +static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg); + +static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg); + +static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg); + +static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg); + +static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg); + +static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg); + #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index f410c8cf6ebf8f71b4585a72568afc88f80ac15d..2ee5e0109c447f37a87eff474d2255cb33f1bde7 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -41,26 +41,26 @@ typedef enum ESyncMessageType { typedef struct SyncPing { ESyncMessageType msgType; const SSyncBuffer *pData; -} SyncPing; +} SyncPing, RaftPing; typedef struct SyncPingReply { ESyncMessageType msgType; const SSyncBuffer *pData; -} SyncPingReply; +} SyncPingReply, RaftPingReply; typedef struct SyncClientRequest { ESyncMessageType msgType; const SSyncBuffer *pData; int64_t seqNum; bool isWeak; -} SyncClientRequest; +} SyncClientRequest, RaftClientRequest; typedef struct SyncClientRequestReply { ESyncMessageType msgType; int32_t errCode; const SSyncBuffer *pErrMsg; const SSyncBuffer *pLeaderHint; -} SyncClientRequestReply; +} SyncClientRequestReply, RaftClientRequestReply; typedef struct SyncRequestVote { ESyncMessageType msgType; @@ -69,7 +69,7 @@ typedef struct SyncRequestVote { SyncGroupId vgId; SyncIndex lastLogIndex; SyncTerm lastLogTerm; -} SyncRequestVote; +} SyncRequestVote, RaftRequestVote; typedef struct SyncRequestVoteReply { ESyncMessageType msgType; @@ -77,7 +77,7 @@ typedef struct SyncRequestVoteReply { SyncNodeId nodeId; SyncGroupId vgId; bool voteGranted; -} SyncRequestVoteReply; +} SyncRequestVoteReply, RaftRequestVoteReply; typedef struct SyncAppendEntries { ESyncMessageType msgType; @@ -88,7 +88,7 @@ typedef struct SyncAppendEntries { int32_t entryCount; SSyncRaftEntry * logEntries; SyncIndex commitIndex; -} SyncAppendEntries; +} SyncAppendEntries, RaftAppendEntries; typedef struct SyncAppendEntriesReply { ESyncMessageType msgType; @@ -96,7 +96,7 @@ typedef struct SyncAppendEntriesReply { SyncNodeId nodeId; bool success; SyncIndex matchIndex; -} SyncAppendEntriesReply; +} SyncAppendEntriesReply, RaftAppendEntriesReply; #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaft.h b/source/libs/sync/inc/syncRaft.h index 0c7e573572230e640094f734ca716886c26ac40c..a247a29fc4dbfe6050433e29c0dbe84483f0e04a 100644 --- a/source/libs/sync/inc/syncRaft.h +++ b/source/libs/sync/inc/syncRaft.h @@ -28,14 +28,56 @@ extern "C" { #include "taosdef.h" typedef struct SRaftId { - SyncNodeId nodeId; + SyncNodeId addr; SyncGroupId vgId; } SRaftId; typedef struct SRaft { - SRaftId id; + SRaftId id; + SSyncFSM* pFsm; + + int32_t (*FpPing)(struct SRaft* ths, const RaftPing* pMsg); + + int32_t (*FpOnPing)(struct SRaft* ths, RaftPing* pMsg); + + int32_t (*FpOnPingReply)(struct SRaft* ths, RaftPingReply* pMsg); + + int32_t (*FpRequestVote)(struct SRaft* ths, const RaftRequestVote* pMsg); + + int32_t (*FpOnRequestVote)(struct SRaft* ths, RaftRequestVote* pMsg); + + int32_t (*FpOnRequestVoteReply)(struct SRaft* ths, RaftRequestVoteReply* pMsg); + + int32_t (*FpAppendEntries)(struct SRaft* ths, const RaftAppendEntries* pMsg); + + int32_t (*FpOnAppendEntries)(struct SRaft* ths, RaftAppendEntries* pMsg); + + int32_t (*FpOnAppendEntriesReply)(struct SRaft* ths, RaftAppendEntriesReply* pMsg); + } SRaft; +SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm); + +void raftClose(SRaft* pRaft); + +static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg); + +static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg); + +static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg); + +static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg); + +static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg); + +static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg); + +static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg); + +static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg); + +static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg); + int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak); static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft); diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index adc82f2c5d9f2db8d7f1cd3f734b7bb129f6173a..516bef4d48c4f9b26d202f6999c1eec78644b056 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_LIBS_TPL_H -#define _TD_LIBS_TPL_H +#ifndef _TD_LIBS_SYNC_RAFT_ENTRY_H +#define _TD_LIBS_SYNC_RAFT_ENTRY_H #ifdef __cplusplus extern "C" { @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" typedef struct SSyncRaftEntry { @@ -37,4 +37,4 @@ typedef struct SSyncRaftEntry { } #endif -#endif /*_TD_LIBS_TPL_H*/ +#endif /*_TD_LIBS_SYNC_RAFT_ENTRY_H*/ diff --git a/source/libs/sync/inc/syncRaftLog.h b/source/libs/sync/inc/syncRaftLog.h index 8c4b5116ea17b2c1c57d0156a165d3bab845594e..ee971062cf7ed5140856f0c5d4483adf157d9d0a 100644 --- a/source/libs/sync/inc/syncRaftLog.h +++ b/source/libs/sync/inc/syncRaftLog.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "taosdef.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf); diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 4cb852f34a5d962824a91e8b7ca666bcb1f130c0..0fdbd7a15084193ec5deef4bc1ae2b7ff008c71a 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -23,17 +23,36 @@ extern "C" { #include #include #include -#include "sync.h" +#include "cJSON.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" -int32_t currentTerm(SyncTerm *pCurrentTerm); +#define RAFT_STORE_BLOCK_SIZE 512 +#define RAFT_STORE_PATH_LEN 128 -int32_t persistCurrentTerm(SyncTerm currentTerm); +typedef struct SRaftStore { + SyncTerm currentTerm; + SRaftId voteFor; + //FileFd fd; + char path[RAFT_STORE_PATH_LEN]; +} SRaftStore; -int32_t voteFor(SRaftId *pRaftId); +SRaftStore *raftStoreOpen(const char *path); -int32_t persistVoteFor(SRaftId *pRaftId); +static int32_t raftStoreInit(SRaftStore *pRaftStore); + +int32_t raftStoreClose(SRaftStore *pRaftStore); + +int32_t raftStorePersist(SRaftStore *pRaftStore); + +static bool raftStoreFileExist(char *path); + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); + +void raftStorePrint(SRaftStore *pRaftStore); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 40c5ff790b6c30ee141633cabd3d730275a026b8..7f97ae9e49cb516dbbc04f082a4068165208bd6b 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "taosdef.h" #ifdef __cplusplus diff --git a/source/libs/sync/inc/syncRequestVote.h b/source/libs/sync/inc/syncRequestVote.h index 3ff96bbe8fad9f80db711e556ec841f7b6dd3082..c2eca55151e24aceae235e7232faf03bb5784894 100644 --- a/source/libs/sync/inc/syncRequestVote.h +++ b/source/libs/sync/inc/syncRequestVote.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncRequestVoteReply.h b/source/libs/sync/inc/syncRequestVoteReply.h index 033ac89bc2f4d0f74afd4a497e77a098b5934159..38068dd0e2db38750a352b4546cf19c49e5eb09b 100644 --- a/source/libs/sync/inc/syncRequestVoteReply.h +++ b/source/libs/sync/inc/syncRequestVoteReply.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 3b6121578ad99b55af5bc81bf9601a48b1391c68..89fcb230fbe06f83a44cdd178bece09f80c5b587 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -23,7 +23,7 @@ extern "C" { #include #include #include -#include "sync.h" +#include "syncInt.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/source/libs/sync/inc/syncTimeout.h b/source/libs/sync/inc/syncTimeout.h index 8159d2566c740296ca58f6d1f8ba05d12d044a34..d9d6a1793917cd2e52275bc074d05523438a6f6b 100644 --- a/source/libs/sync/inc/syncTimeout.h +++ b/source/libs/sync/inc/syncTimeout.h @@ -23,6 +23,7 @@ extern "C" { #include #include #include +#include "syncInt.h" #include "syncMessage.h" #include "syncRaft.h" #include "taosdef.h" diff --git a/include/common/ttimezone.h b/source/libs/sync/inc/syncVoteMgr.h similarity index 77% rename from include/common/ttimezone.h rename to source/libs/sync/inc/syncVoteMgr.h index cc50a29d57a2f1a308b8b236fc7bb80122adb387..cfcf58bee2199735ed6058d563012a6e1e364248 100644 --- a/include/common/ttimezone.h +++ b/source/libs/sync/inc/syncVoteMgr.h @@ -13,17 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_COMMON_TIMEZONE_H_ -#define _TD_COMMON_TIMEZONE_H_ +#ifndef _TD_LIBS_SYNC_VOTG_MGR_H +#define _TD_LIBS_SYNC_VOTG_MGR_H #ifdef __cplusplus extern "C" { #endif -void tsSetTimeZone(); +#include +#include +#include +#include "syncInt.h" +#include "taosdef.h" #ifdef __cplusplus } #endif -#endif /*_TD_COMMON_TIMEZONE_H_*/ +#endif /*_TD_LIBS_SYNC_VOTG_MGR_H*/ diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 1286108664270b81a4b5ae70f6e592dc67a6a5ad..2b9c59ec92a3b368426af4000b93eb6db1858727 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -14,104 +14,98 @@ */ #include "syncAppendEntries.h" -#include "sync.h" void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { - -// TLA+ Spec -//AppendEntries(i, j) == -// /\ i /= j -// /\ state[i] = Leader -// /\ LET prevLogIndex == nextIndex[i][j] - 1 -// prevLogTerm == IF prevLogIndex > 0 THEN -// log[i][prevLogIndex].term -// ELSE -// 0 -// \* Send up to 1 entry, constrained by the end of the log. -// lastEntry == Min({Len(log[i]), nextIndex[i][j]}) -// entries == SubSeq(log[i], nextIndex[i][j], lastEntry) -// IN Send([mtype |-> AppendEntriesRequest, -// mterm |-> currentTerm[i], -// mprevLogIndex |-> prevLogIndex, -// mprevLogTerm |-> prevLogTerm, -// mentries |-> entries, -// \* mlog is used as a history variable for the proof. -// \* It would not exist in a real implementation. -// mlog |-> log[i], -// mcommitIndex |-> Min({commitIndex[i], lastEntry}), -// msource |-> i, -// mdest |-> j]) -// /\ UNCHANGED <> - + // TLA+ Spec + // AppendEntries(i, j) == + // /\ i /= j + // /\ state[i] = Leader + // /\ LET prevLogIndex == nextIndex[i][j] - 1 + // prevLogTerm == IF prevLogIndex > 0 THEN + // log[i][prevLogIndex].term + // ELSE + // 0 + // \* Send up to 1 entry, constrained by the end of the log. + // lastEntry == Min({Len(log[i]), nextIndex[i][j]}) + // entries == SubSeq(log[i], nextIndex[i][j], lastEntry) + // IN Send([mtype |-> AppendEntriesRequest, + // mterm |-> currentTerm[i], + // mprevLogIndex |-> prevLogIndex, + // mprevLogTerm |-> prevLogTerm, + // mentries |-> entries, + // \* mlog is used as a history variable for the proof. + // \* It would not exist in a real implementation. + // mlog |-> log[i], + // mcommitIndex |-> Min({commitIndex[i], lastEntry}), + // msource |-> i, + // mdest |-> j]) + // /\ UNCHANGED <> } void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) { - -// TLA+ Spec -//HandleAppendEntriesRequest(i, j, m) == -// LET logOk == \/ m.mprevLogIndex = 0 -// \/ /\ m.mprevLogIndex > 0 -// /\ m.mprevLogIndex <= Len(log[i]) -// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term -// IN /\ m.mterm <= currentTerm[i] -// /\ \/ /\ \* reject request -// \/ m.mterm < currentTerm[i] -// \/ /\ m.mterm = currentTerm[i] -// /\ state[i] = Follower -// /\ \lnot logOk -// /\ Reply([mtype |-> AppendEntriesResponse, -// mterm |-> currentTerm[i], -// msuccess |-> FALSE, -// mmatchIndex |-> 0, -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> -// \/ \* return to follower state -// /\ m.mterm = currentTerm[i] -// /\ state[i] = Candidate -// /\ state' = [state EXCEPT ![i] = Follower] -// /\ UNCHANGED <> -// \/ \* accept request -// /\ m.mterm = currentTerm[i] -// /\ state[i] = Follower -// /\ logOk -// /\ LET index == m.mprevLogIndex + 1 -// IN \/ \* already done with request -// /\ \/ m.mentries = << >> -// \/ /\ m.mentries /= << >> -// /\ Len(log[i]) >= index -// /\ log[i][index].term = m.mentries[1].term -// \* This could make our commitIndex decrease (for -// \* example if we process an old, duplicated request), -// \* but that doesn't really affect anything. -// /\ commitIndex' = [commitIndex EXCEPT ![i] = -// m.mcommitIndex] -// /\ Reply([mtype |-> AppendEntriesResponse, -// mterm |-> currentTerm[i], -// msuccess |-> TRUE, -// mmatchIndex |-> m.mprevLogIndex + -// Len(m.mentries), -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> -// \/ \* conflict: remove 1 entry -// /\ m.mentries /= << >> -// /\ Len(log[i]) >= index -// /\ log[i][index].term /= m.mentries[1].term -// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> -// log[i][index2]] -// IN log' = [log EXCEPT ![i] = new] -// /\ UNCHANGED <> -// \/ \* no conflict: append entry -// /\ m.mentries /= << >> -// /\ Len(log[i]) = m.mprevLogIndex -// /\ log' = [log EXCEPT ![i] = -// Append(log[i], m.mentries[1])] -// /\ UNCHANGED <> -// /\ UNCHANGED <> -// - - + // TLA+ Spec + // HandleAppendEntriesRequest(i, j, m) == + // LET logOk == \/ m.mprevLogIndex = 0 + // \/ /\ m.mprevLogIndex > 0 + // /\ m.mprevLogIndex <= Len(log[i]) + // /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term + // IN /\ m.mterm <= currentTerm[i] + // /\ \/ /\ \* reject request + // \/ m.mterm < currentTerm[i] + // \/ /\ m.mterm = currentTerm[i] + // /\ state[i] = Follower + // /\ \lnot logOk + // /\ Reply([mtype |-> AppendEntriesResponse, + // mterm |-> currentTerm[i], + // msuccess |-> FALSE, + // mmatchIndex |-> 0, + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> + // \/ \* return to follower state + // /\ m.mterm = currentTerm[i] + // /\ state[i] = Candidate + // /\ state' = [state EXCEPT ![i] = Follower] + // /\ UNCHANGED <> + // \/ \* accept request + // /\ m.mterm = currentTerm[i] + // /\ state[i] = Follower + // /\ logOk + // /\ LET index == m.mprevLogIndex + 1 + // IN \/ \* already done with request + // /\ \/ m.mentries = << >> + // \/ /\ m.mentries /= << >> + // /\ Len(log[i]) >= index + // /\ log[i][index].term = m.mentries[1].term + // \* This could make our commitIndex decrease (for + // \* example if we process an old, duplicated request), + // \* but that doesn't really affect anything. + // /\ commitIndex' = [commitIndex EXCEPT ![i] = + // m.mcommitIndex] + // /\ Reply([mtype |-> AppendEntriesResponse, + // mterm |-> currentTerm[i], + // msuccess |-> TRUE, + // mmatchIndex |-> m.mprevLogIndex + + // Len(m.mentries), + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> + // \/ \* conflict: remove 1 entry + // /\ m.mentries /= << >> + // /\ Len(log[i]) >= index + // /\ log[i][index].term /= m.mentries[1].term + // /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |-> + // log[i][index2]] + // IN log' = [log EXCEPT ![i] = new] + // /\ UNCHANGED <> + // \/ \* no conflict: append entry + // /\ m.mentries /= << >> + // /\ Len(log[i]) = m.mprevLogIndex + // /\ log' = [log EXCEPT ![i] = + // Append(log[i], m.mentries[1])] + // /\ UNCHANGED <> + // /\ UNCHANGED <> + // } diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 4a9055e172fce1aad4753410301c34f4c8823275..05734237b93de6ec43e85116e0d65640f0ccd942 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -14,21 +14,18 @@ */ #include "syncAppendEntriesReply.h" -#include "sync.h" void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) { - -// TLA+ Spec -//HandleAppendEntriesResponse(i, j, m) == -// /\ m.mterm = currentTerm[i] -// /\ \/ /\ m.msuccess \* successful -// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] -// /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] -// \/ /\ \lnot m.msuccess \* not successful -// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = -// Max({nextIndex[i][j] - 1, 1})] -// /\ UNCHANGED <> -// /\ Discard(m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleAppendEntriesResponse(i, j, m) == + // /\ m.mterm = currentTerm[i] + // /\ \/ /\ m.msuccess \* successful + // /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + // /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + // \/ /\ \lnot m.msuccess \* not successful + // /\ nextIndex' = [nextIndex EXCEPT ![i][j] = + // Max({nextIndex[i][j] - 1, 1})] + // /\ UNCHANGED <> + // /\ Discard(m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 738fc4c5e129562d368451c3f9d1eb3315060bb4..329105e2a1214aba3cfb1bce2850f292f85811b8 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncElection.h" diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c new file mode 100644 index 0000000000000000000000000000000000000000..e71cf55cb193c4716bba452f1e5601a64b559fad --- /dev/null +++ b/source/libs/sync/src/syncEnv.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncEnv.h" +#include + +SSyncEnv *gSyncEnv = NULL; + +int32_t syncEnvStart() { + int32_t ret; + gSyncEnv = (SSyncEnv *)malloc(sizeof(SSyncEnv)); + assert(gSyncEnv != NULL); + ret = doSyncEnvStart(gSyncEnv); + return ret; +} + +int32_t syncEnvStop() { + int32_t ret = doSyncEnvStop(gSyncEnv); + return ret; +} + +static int32_t doSyncEnvStart(SSyncEnv *pSyncEnv) { return 0; } + +static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { return 0; } diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c new file mode 100644 index 0000000000000000000000000000000000000000..bb20d11e37b3edc56f2958a5021f5881129e220d --- /dev/null +++ b/source/libs/sync/src/syncIO.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "syncIO.h" +#include +#include "syncOnMessage.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tutil.h" + +SSyncIO *gSyncIO = NULL; + +int32_t syncIOSendMsg(void *handle, const SEpSet *pEpSet, SRpcMsg *pMsg) { return 0; } + +int32_t syncIOStart() { return 0; } + +int32_t syncIOStop() { return 0; } + +static void syncTick(void *param, void *tmrId) { + SSyncIO *io = (SSyncIO *)param; + sDebug("syncTick ... "); + + SRpcMsg rpcMsg; + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "TICK"); + rpcMsg.contLen = 10; + rpcMsg.handle = NULL; + rpcMsg.msgType = 2; + + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg)); + + taosWriteQitem(io->pMsgQ, pTemp); + + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); +} + +void *syncConsumer(void *param) { + SSyncIO *io = param; + + STaosQall *qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + + qall = taosAllocateQall(); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(io->pQset, qall, NULL, NULL); + sDebug("%d sync-io msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + sDebug("sync-io recv type:%d msg:%s", pRpcMsg->msgType, (char *)(pRpcMsg->pCont)); + } + + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; ++i) { + taosGetQitem(qall, (void **)&pRpcMsg); + rpcFreeCont(pRpcMsg->pCont); + + if (pRpcMsg->handle != NULL) { + int msgSize = 128; + memset(&rpcMsg, 0, sizeof(rpcMsg)); + rpcMsg.pCont = rpcMallocCont(msgSize); + rpcMsg.contLen = msgSize; + rpcMsg.handle = pRpcMsg->handle; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } + + taosFreeQitem(pRpcMsg); + } + } + + taosFreeQall(qall); + return NULL; +} + +static int retrieveAuthInfo(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey) { + // app shall retrieve the auth info based on meterID from DB or a data file + // demo code here only for simple demo + int ret = 0; + return ret; +} + +static void processResponse(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + sDebug("processResponse ... "); + rpcFreeCont(pMsg->pCont); +} + +static void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SSyncIO *io = pParent; + SRpcMsg *pTemp; + + pTemp = taosAllocateQitem(sizeof(SRpcMsg)); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + sDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); + taosWriteQitem(io->pMsgQ, pTemp); +} + +SSyncIO *syncIOCreate() { + SSyncIO *io = (SSyncIO *)malloc(sizeof(SSyncIO)); + memset(io, 0, sizeof(*io)); + + io->pMsgQ = taosOpenQueue(); + io->pQset = taosOpenQset(); + taosAddIntoQset(io->pQset, io->pMsgQ, NULL); + + io->start = doSyncIOStart; + io->stop = doSyncIOStop; + io->ping = doSyncIOPing; + io->onMsg = doSyncIOOnMsg; + io->destroy = doSyncIODestroy; + + return io; +} + +static int32_t doSyncIOStart(SSyncIO *io) { + taosBlockSIGPIPE(); + + tsRpcForceTcp = 1; + + // cient rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = "SYNC-IO-CLIENT"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processResponse; + rpcInit.sessions = 100; + rpcInit.idleTime = 100; + rpcInit.user = "sync-io"; + rpcInit.secret = "sync-io"; + rpcInit.ckey = "key"; + rpcInit.spi = 0; + rpcInit.connType = TAOS_CONN_CLIENT; + + io->clientRpc = rpcOpen(&rpcInit); + if (io->clientRpc == NULL) { + sError("failed to initialize RPC"); + return -1; + } + } + + // server rpc init + { + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 38000; + rpcInit.label = "SYNC-IO-SERVER"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processRequestMsg; + rpcInit.sessions = 1000; + rpcInit.idleTime = 2 * 1500; + rpcInit.afp = retrieveAuthInfo; + rpcInit.parent = io; + rpcInit.connType = TAOS_CONN_SERVER; + + void *pRpc = rpcOpen(&rpcInit); + if (pRpc == NULL) { + sError("failed to start RPC server"); + return -1; + } + } + + io->epSet.inUse = 0; + addEpIntoEpSet(&io->epSet, "127.0.0.1", 38000); + + // start consumer thread + { + if (pthread_create(&io->tid, NULL, syncConsumer, io) != 0) { + sError("failed to create sync consumer thread since %s", strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + // start tmr thread + io->syncTimerManager = taosTmrInit(1000, 50, 10000, "SYNC"); + io->syncTimer = taosTmrStart(syncTick, 1000, io, io->syncTimerManager); + + return 0; +} + +static int32_t doSyncIOStop(SSyncIO *io) { + atomic_store_8(&io->isStart, 0); + pthread_join(io->tid, NULL); + return 0; +} + +static int32_t doSyncIOPing(SSyncIO *io) { + SRpcMsg rpcMsg, rspMsg; + + rpcMsg.pCont = rpcMallocCont(10); + snprintf(rpcMsg.pCont, 10, "ping"); + rpcMsg.contLen = 10; + rpcMsg.handle = NULL; + rpcMsg.msgType = 1; + + rpcSendRequest(io->clientRpc, &io->epSet, &rpcMsg, NULL); + + return 0; +} + +static int32_t doSyncIOOnMsg(struct SSyncIO *io, void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { return 0; } + +static int32_t doSyncIODestroy(SSyncIO *io) { + int8_t start = atomic_load_8(&io->isStart); + assert(start == 0); + + if (io->serverRpc != NULL) { + free(io->serverRpc); + io->serverRpc = NULL; + } + + if (io->clientRpc != NULL) { + free(io->clientRpc); + io->clientRpc = NULL; + } + + if (io->pMsgQ != NULL) { + free(io->pMsgQ); + io->pMsgQ = NULL; + } + + if (io->pQset != NULL) { + free(io->pQset); + io->pQset = NULL; + } + + return 0; +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index fbb969eb1c91b6d0641dd1ff8d02b49f4ebe428c..bd2952505ed1e28b36da0333dc31e97eae3ae4eb 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -16,12 +16,17 @@ #include #include "sync.h" #include "syncInt.h" +#include "syncRaft.h" int32_t syncInit() { return 0; } void syncCleanUp() {} -int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; } +int64_t syncStart(const SSyncInfo* pSyncInfo) { + SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); + assert(pSyncNode != NULL); + return 0; +} void syncStop(int64_t rid) {} @@ -31,4 +36,74 @@ int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { r ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; } -void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} \ No newline at end of file +void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {} + +SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { + SSyncNode* pSyncNode = (SSyncNode*)malloc(sizeof(SSyncNode)); + assert(pSyncNode != NULL); + + pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg; + + pSyncNode->FpPing = doSyncNodePing; + pSyncNode->FpOnPing = onSyncNodePing; + pSyncNode->FpOnPingReply = onSyncNodePingReply; + pSyncNode->FpRequestVote = doSyncNodeRequestVote; + pSyncNode->FpOnRequestVote = onSyncNodeRequestVote; + pSyncNode->FpOnRequestVoteReply = onSyncNodeRequestVoteReply; + pSyncNode->FpAppendEntries = doSyncNodeAppendEntries; + pSyncNode->FpOnAppendEntries = onSyncNodeAppendEntries; + pSyncNode->FpOnAppendEntriesReply = onSyncNodeAppendEntriesReply; + + return pSyncNode; +} + +void syncNodeClose(SSyncNode* pSyncNode) { + assert(pSyncNode != NULL); + raftClose(pSyncNode->pRaft); + free(pSyncNode); +} + +static int32_t doSyncNodePing(struct SSyncNode* ths, const SyncPing* pMsg) { + int32_t ret = ths->pRaft->FpPing(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodePing(struct SSyncNode* ths, SyncPing* pMsg) { + int32_t ret = ths->pRaft->FpOnPing(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodePingReply(struct SSyncNode* ths, SyncPingReply* pMsg) { + int32_t ret = ths->pRaft->FpOnPingReply(ths->pRaft, pMsg); + return ret; +} + +static int32_t doSyncNodeRequestVote(struct SSyncNode* ths, const SyncRequestVote* pMsg) { + int32_t ret = ths->pRaft->FpRequestVote(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeRequestVote(struct SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = ths->pRaft->FpOnRequestVote(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeRequestVoteReply(struct SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = ths->pRaft->FpOnRequestVoteReply(ths->pRaft, pMsg); + return ret; +} + +static int32_t doSyncNodeAppendEntries(struct SSyncNode* ths, const SyncAppendEntries* pMsg) { + int32_t ret = ths->pRaft->FpAppendEntries(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeAppendEntries(struct SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = ths->pRaft->FpOnAppendEntries(ths->pRaft, pMsg); + return ret; +} + +static int32_t onSyncNodeAppendEntriesReply(struct SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = ths->pRaft->FpOnAppendEntriesReply(ths->pRaft, pMsg); + return ret; +} \ No newline at end of file diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index dcfc940f76b37570f6052ef47bf15aa87dbb698f..89373037250aaf22d0c3f0ef06d8f9c27409d59c 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -14,7 +14,6 @@ */ #include "syncMessage.h" -#include "sync.h" #include "syncRaft.h" void onMessage(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/source/libs/sync/src/syncOnMessage.c b/source/libs/sync/src/syncOnMessage.c index 738fc4c5e129562d368451c3f9d1eb3315060bb4..19a97ee1566d32ed0fc8786c4d75542c2c435f30 100644 --- a/source/libs/sync/src/syncOnMessage.c +++ b/source/libs/sync/src/syncOnMessage.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncOnMessage.h" diff --git a/source/libs/sync/src/syncRaft.c b/source/libs/sync/src/syncRaft.c index 85c2c6fe27ba70b4ecdac85c5d31aac165ce0a00..9f139730d102f4d6ab414938042711af996d65df 100644 --- a/source/libs/sync/src/syncRaft.c +++ b/source/libs/sync/src/syncRaft.c @@ -16,6 +16,51 @@ #include "syncRaft.h" #include "sync.h" +SRaft* raftOpen(SRaftId raftId, SSyncFSM* pFsm) { + SRaft* pRaft = (SRaft*)malloc(sizeof(SRaft)); + assert(pRaft != NULL); + + pRaft->id = raftId; + pRaft->pFsm = pFsm; + + pRaft->FpPing = doRaftPing; + pRaft->FpOnPing = onRaftPing; + pRaft->FpOnPingReply = onRaftPingReply; + + pRaft->FpRequestVote = doRaftRequestVote; + pRaft->FpOnRequestVote = onRaftRequestVote; + pRaft->FpOnRequestVoteReply = onRaftRequestVoteReply; + + pRaft->FpAppendEntries = doRaftAppendEntries; + pRaft->FpOnAppendEntries = onRaftAppendEntries; + pRaft->FpOnAppendEntriesReply = onRaftAppendEntriesReply; + + return pRaft; +} + +void raftClose(SRaft* pRaft) { + assert(pRaft != NULL); + free(pRaft); +} + +static int32_t doRaftPing(struct SRaft* ths, const RaftPing* pMsg) { return 0; } + +static int32_t onRaftPing(struct SRaft* ths, RaftPing* pMsg) { return 0; } + +static int32_t onRaftPingReply(struct SRaft* ths, RaftPingReply* pMsg) { return 0; } + +static int32_t doRaftRequestVote(struct SRaft* ths, const RaftRequestVote* pMsg) { return 0; } + +static int32_t onRaftRequestVote(struct SRaft* ths, RaftRequestVote* pMsg) { return 0; } + +static int32_t onRaftRequestVoteReply(struct SRaft* ths, RaftRequestVoteReply* pMsg) { return 0; } + +static int32_t doRaftAppendEntries(struct SRaft* ths, const RaftAppendEntries* pMsg) { return 0; } + +static int32_t onRaftAppendEntries(struct SRaft* ths, RaftAppendEntries* pMsg) { return 0; } + +static int32_t onRaftAppendEntriesReply(struct SRaft* ths, RaftAppendEntriesReply* pMsg) { return 0; } + int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; } static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; } diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 738fc4c5e129562d368451c3f9d1eb3315060bb4..e525d3c7c26366aa5f64eb596ee0ad3e92ad9340 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncRaftEntry.h" diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 4a5fc201b07c01cf537f809064ad10618115b4d6..37bb3ce48cc5f7651a8c83d6457da12f11e9d42f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -14,7 +14,6 @@ */ #include "syncRaftLog.h" -#include "sync.h" int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index d45e53132cbc6d14f70f2c9d09948b9f09d283ce..964cc784905e5f22171b50413f9f78fc14a328de 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -14,12 +14,142 @@ */ #include "syncRaftStore.h" -#include "sync.h" +#include "cJSON.h" -int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; } +// to complie success: FileIO interface is modified -int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; } +SRaftStore *raftStoreOpen(const char *path) { return NULL;} -int32_t voteFor(SRaftId *pRaftId) { return 0; } +static int32_t raftStoreInit(SRaftStore *pRaftStore) { return 0;} -int32_t persistVoteFor(SRaftId *pRaftId) { return 0; } \ No newline at end of file +int32_t raftStoreClose(SRaftStore *pRaftStore) { return 0;} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0;} + +static bool raftStoreFileExist(char *path) { return 0;} + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { return 0;} + +void raftStorePrint(SRaftStore *pRaftStore) {} + + + +#if 0 + +SRaftStore *raftStoreOpen(const char *path) { + int32_t ret; + + SRaftStore *pRaftStore = malloc(sizeof(SRaftStore)); + if (pRaftStore == NULL) { + sError("raftStoreOpen malloc error"); + return NULL; + } + memset(pRaftStore, 0, sizeof(*pRaftStore)); + snprintf(pRaftStore->path, sizeof(pRaftStore->path), "%s", path); + + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + memset(storeBuf, 0, sizeof(storeBuf)); + + if (!raftStoreFileExist(pRaftStore->path)) { + ret = raftStoreInit(pRaftStore); + assert(ret == 0); + } + + pRaftStore->fd = taosOpenFileReadWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return NULL; + } + + int len = taosReadFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(len == RAFT_STORE_BLOCK_SIZE); + + ret = raftStoreDeserialize(pRaftStore, storeBuf, len); + assert(ret == 0); + + return pRaftStore; +} + +static int32_t raftStoreInit(SRaftStore *pRaftStore) { + pRaftStore->fd = taosOpenFileCreateWrite(pRaftStore->path); + if (pRaftStore->fd < 0) { + return -1; + } + + pRaftStore->currentTerm = 0; + pRaftStore->voteFor.addr = 0; + pRaftStore->voteFor.vgId = 0; + + int32_t ret = raftStorePersist(pRaftStore); + assert(ret == 0); + + taosCloseFile(pRaftStore->fd); + return 0; +} + +int32_t raftStoreClose(SRaftStore *pRaftStore) { + taosCloseFile(pRaftStore->fd); + free(pRaftStore); + return 0; +} + +int32_t raftStorePersist(SRaftStore *pRaftStore) { + int32_t ret; + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + + ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + assert(ret == 0); + + taosLSeekFile(pRaftStore->fd, 0, SEEK_SET); + + ret = taosWriteFile(pRaftStore->fd, storeBuf, sizeof(storeBuf)); + assert(ret == RAFT_STORE_BLOCK_SIZE); + + fsync(pRaftStore->fd); + return 0; +} + +static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } + +int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { + cJSON *pRoot = cJSON_CreateObject(); + cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); + cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + + char *serialized = cJSON_Print(pRoot); + int len2 = strlen(serialized); + assert(len2 < len); + memset(buf, 0, len); + snprintf(buf, len, "%s", serialized); + free(serialized); + + cJSON_Delete(pRoot); + return 0; +} + +int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { + assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); + cJSON *pRoot = cJSON_Parse(buf); + + cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); + pRaftStore->currentTerm = pCurrentTerm->valueint; + + cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); + pRaftStore->voteFor.addr = pVoteForAddr->valueint; + + cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); + pRaftStore->voteFor.vgId = pVoteForVgid->valueint; + + cJSON_Delete(pRoot); + return 0; +} + +void raftStorePrint(SRaftStore *pRaftStore) { + char storeBuf[RAFT_STORE_BLOCK_SIZE]; + raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); + printf("%s\n", storeBuf); +} + +#endif diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 738fc4c5e129562d368451c3f9d1eb3315060bb4..4cea7c150e19b490910adf771cfca050c03aea22 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -13,4 +13,4 @@ * along with this program. If not, see . */ -#include "sync.h" +#include "syncReplication.h" diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index c31ec0f34d521b214b7d45a3ce32c3890ebcabd5..7aee47b8e42ced91522c177d887785a6193328af 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -14,46 +14,41 @@ */ #include "syncRequestVote.h" -#include "sync.h" void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { - -// TLA+ Spec -//RequestVote(i, j) == -// /\ state[i] = Candidate -// /\ j \notin votesResponded[i] -// /\ Send([mtype |-> RequestVoteRequest, -// mterm |-> currentTerm[i], -// mlastLogTerm |-> LastTerm(log[i]), -// mlastLogIndex |-> Len(log[i]), -// msource |-> i, -// mdest |-> j]) -// /\ UNCHANGED <> - + // TLA+ Spec + // RequestVote(i, j) == + // /\ state[i] = Candidate + // /\ j \notin votesResponded[i] + // /\ Send([mtype |-> RequestVoteRequest, + // mterm |-> currentTerm[i], + // mlastLogTerm |-> LastTerm(log[i]), + // mlastLogIndex |-> Len(log[i]), + // msource |-> i, + // mdest |-> j]) + // /\ UNCHANGED <> } void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg) { - -// TLA+ Spec -//HandleRequestVoteRequest(i, j, m) == -// LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) -// \/ /\ m.mlastLogTerm = LastTerm(log[i]) -// /\ m.mlastLogIndex >= Len(log[i]) -// grant == /\ m.mterm = currentTerm[i] -// /\ logOk -// /\ votedFor[i] \in {Nil, j} -// IN /\ m.mterm <= currentTerm[i] -// /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] -// \/ ~grant /\ UNCHANGED votedFor -// /\ Reply([mtype |-> RequestVoteResponse, -// mterm |-> currentTerm[i], -// mvoteGranted |-> grant, -// \* mlog is used just for the `elections' history variable for -// \* the proof. It would not exist in a real implementation. -// mlog |-> log[i], -// msource |-> i, -// mdest |-> j], -// m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleRequestVoteRequest(i, j, m) == + // LET logOk == \/ m.mlastLogTerm > LastTerm(log[i]) + // \/ /\ m.mlastLogTerm = LastTerm(log[i]) + // /\ m.mlastLogIndex >= Len(log[i]) + // grant == /\ m.mterm = currentTerm[i] + // /\ logOk + // /\ votedFor[i] \in {Nil, j} + // IN /\ m.mterm <= currentTerm[i] + // /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] + // \/ ~grant /\ UNCHANGED votedFor + // /\ Reply([mtype |-> RequestVoteResponse, + // mterm |-> currentTerm[i], + // mvoteGranted |-> grant, + // \* mlog is used just for the `elections' history variable for + // \* the proof. It would not exist in a real implementation. + // mlog |-> log[i], + // msource |-> i, + // mdest |-> j], + // m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index ba9787f00c156a7b205a87108dd5670ff0f921b3..a9c88a797545cc207d94c338ef9bc0f86f77e5c4 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -14,25 +14,22 @@ */ #include "syncRequestVoteReply.h" -#include "sync.h" void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) { - -// TLA+ Spec -//HandleRequestVoteResponse(i, j, m) == -// \* This tallies votes even when the current state is not Candidate, but -// \* they won't be looked at, so it doesn't matter. -// /\ m.mterm = currentTerm[i] -// /\ votesResponded' = [votesResponded EXCEPT ![i] = -// votesResponded[i] \cup {j}] -// /\ \/ /\ m.mvoteGranted -// /\ votesGranted' = [votesGranted EXCEPT ![i] = -// votesGranted[i] \cup {j}] -// /\ voterLog' = [voterLog EXCEPT ![i] = -// voterLog[i] @@ (j :> m.mlog)] -// \/ /\ ~m.mvoteGranted -// /\ UNCHANGED <> -// /\ Discard(m) -// /\ UNCHANGED <> - + // TLA+ Spec + // HandleRequestVoteResponse(i, j, m) == + // \* This tallies votes even when the current state is not Candidate, but + // \* they won't be looked at, so it doesn't matter. + // /\ m.mterm = currentTerm[i] + // /\ votesResponded' = [votesResponded EXCEPT ![i] = + // votesResponded[i] \cup {j}] + // /\ \/ /\ m.mvoteGranted + // /\ votesGranted' = [votesGranted EXCEPT ![i] = + // votesGranted[i] \cup {j}] + // /\ voterLog' = [voterLog EXCEPT ![i] = + // voterLog[i] @@ (j :> m.mlog)] + // \/ /\ ~m.mvoteGranted + // /\ UNCHANGED <> + // /\ Discard(m) + // /\ UNCHANGED <> } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 8a27f097d1335a16d3033d091ba6e5221029a4b8..da194780ffdf17d5b511a2592b736aa927263b18 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -14,7 +14,6 @@ */ #include "syncSnapshot.h" -#include "sync.h" #include "syncRaft.h" int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; } diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 206dd70046b07f5597b96e819d8fd10ceee59433..e27df55d0712d16c6d396f7bb7f0acd6daee30c1 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -14,6 +14,5 @@ */ #include "syncTimeout.h" -#include "sync.h" void onTimeout(SRaft *pRaft, void *pMsg) {} \ No newline at end of file diff --git a/include/common/type/type.h b/source/libs/sync/src/syncVoteMgr.c similarity index 82% rename from include/common/type/type.h rename to source/libs/sync/src/syncVoteMgr.c index 3cbea6edbb41252a303615a4db3c45effed7e2bd..02cf4ac03365eb8cedb49eff0b1ba1de08f4699f 100644 --- a/include/common/type/type.h +++ b/source/libs/sync/src/syncVoteMgr.c @@ -13,15 +13,4 @@ * along with this program. If not, see . */ -#ifndef _TD_TYPE_H_ -#define _TD_TYPE_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TYPE_H_*/ \ No newline at end of file +#include "syncVoteMgr.h" diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..e655ac01be5cef4af5ede6e5d2735bf67eb123c3 --- /dev/null +++ b/source/libs/sync/test/CMakeLists.txt @@ -0,0 +1,55 @@ +add_executable(syncTest "") +add_executable(syncEnvTest "") +add_executable(syncPingTest "") + + +target_sources(syncTest + PRIVATE + "syncTest.cpp" +) +target_sources(syncEnvTest + PRIVATE + "syncEnvTest.cpp" +) +target_sources(syncPingTest + PRIVATE + "syncPingTest.cpp" +) + + +target_include_directories(syncTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncEnvTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncPingTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + + +target_link_libraries(syncTest + sync + gtest_main +) +target_link_libraries(syncEnvTest + sync + gtest_main +) +target_link_libraries(syncPingTest + sync + gtest_main +) + + +enable_testing() +add_test( + NAME sync_test + COMMAND syncTest +) diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..1d050e709410fd9fafdc52151850f0e4e76c0a2b --- /dev/null +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -0,0 +1,56 @@ +#include "syncEnv.h" +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void doSync() { + SSyncInfo syncInfo; + syncInfo.vgId = 1; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); +} + +int main() { + //taosInitLog((char*)"syncEnvTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret = syncIOStart(); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + doSync(); + + while (1) { + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncPingTest.cpp b/source/libs/sync/test/syncPingTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e62d0519463df03247faa877d18bd8c7d03217a7 --- /dev/null +++ b/source/libs/sync/test/syncPingTest.cpp @@ -0,0 +1,65 @@ +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void doSync() { + SSyncFSM* pFsm; + + SSyncInfo syncInfo; + syncInfo.vgId = 1; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping"); + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = 0; + pCfg->replicaNum = 3; + + pCfg->nodeInfo[0].nodePort = 7010; + taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + + pCfg->nodeInfo[1].nodePort = 7110; + taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn); + + pCfg->nodeInfo[2].nodePort = 7210; + taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn); + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnPing = pSyncNode->FpOnPing; + gSyncIO->pSyncNode = pSyncNode; +} + +int main() { + //taosInitLog((char*)"syncPingTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + logTest(); + + int32_t ret = syncIOStart(); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + doSync(); + + while (1) { + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 47566d537e74b4369806f11977220288aff2b00f..0f72fd822f06ccde72d01af1a085b4d609c43533 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -1,7 +1,58 @@ #include +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftStore.h" +#include "gtest/gtest.h" -int main() { - printf("test \n"); - return 0; +void *pingFunc(void *param) { + SSyncIO *io = (SSyncIO *)param; + while (1) { + sDebug("io->ping"); + io->ping(io); + sleep(1); + } + return NULL; } +int main() { + //taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + sTrace("sync log test: trace"); + sDebug("sync log test: debug"); + sInfo("sync log test: info"); + sWarn("sync log test: warn"); + sError("sync log test: error"); + sFatal("sync log test: fatal"); + + SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + // assert(pRaftStore != NULL); + + // raftStorePrint(pRaftStore); + + // pRaftStore->currentTerm = 100; + // pRaftStore->voteFor.addr = 200; + // pRaftStore->voteFor.vgId = 300; + + // raftStorePrint(pRaftStore); + + // raftStorePersist(pRaftStore); + + // sDebug("sync test"); + + // SSyncIO *syncIO = syncIOCreate(); + // assert(syncIO != NULL); + + // syncIO->start(syncIO); + + // sleep(2); + + // pthread_t tid; + // pthread_create(&tid, NULL, pingFunc, syncIO); + + // while (1) { + // sleep(1); + // } + return 0; +} diff --git a/source/libs/tdb/src/db/tdbEnv.c b/source/libs/tdb/src/db/tdbEnv.c index 3670c770abd520fdee6cd8c21d82f3c0939150db..a415d6d60e500cae4389aaf79fdc18f60cae4349 100644 --- a/source/libs/tdb/src/db/tdbEnv.c +++ b/source/libs/tdb/src/db/tdbEnv.c @@ -18,7 +18,7 @@ struct STDbEnv { char * rootDir; // root directory of the environment char * jname; // journal file name - int jfd; // journal file fd + TdFilePtr jpFile; // journal file fd pgsz_t pgSize; // page size cachesz_t cacheSize; // total cache size STDbList dbList; // TDB List @@ -55,7 +55,7 @@ int tdbEnvCreate(TENV **ppEnv, const char *rootDir) { pEnv->rootDir = (char *)(&pEnv[1]); pEnv->jname = pEnv->rootDir + slen + 1; - pEnv->jfd = -1; + pEnv->jpFile = NULL; pEnv->pgSize = TDB_DEFAULT_PGSIZE; pEnv->cacheSize = TDB_DEFAULT_CACHE_SIZE; @@ -139,8 +139,8 @@ static int tdbEnvDestroy(TENV *pEnv) { } int tdbEnvBeginTxn(TENV *pEnv) { - pEnv->jfd = open(pEnv->jname, O_CREAT | O_RDWR, 0755); - if (pEnv->jfd < 0) { + pEnv->jpFile = taosOpenFile(pEnv->jname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pEnv->jpFile == NULL) { return -1; } @@ -149,8 +149,8 @@ int tdbEnvBeginTxn(TENV *pEnv) { int tdbEnvCommit(TENV *pEnv) { /* TODO */ - close(pEnv->jfd); - pEnv->jfd = -1; + taosCloseFile(&pEnv->jpFile); + pEnv->jpFile = NULL; return 0; } diff --git a/source/libs/tdb/src/db/tdbPgFile.c b/source/libs/tdb/src/db/tdbPgFile.c index ee5b486f7b75d42ec87178da0de7603c21aebaf8..12f062ebf767b927172e91a7a86de4e3cae7d32e 100644 --- a/source/libs/tdb/src/db/tdbPgFile.c +++ b/source/libs/tdb/src/db/tdbPgFile.c @@ -51,10 +51,10 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { pPgFile->fname = (char *)(&(pPgFile[1])); memcpy(pPgFile->fname, fname, fnameLen); pPgFile->fname[fnameLen] = '\0'; - pPgFile->fd = -1; + pPgFile->pFile = NULL; - pPgFile->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pPgFile->fd < 0) { + pPgFile->pFile = taosOpenFile(fname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ); + if (pPgFile->pFile == NULL) { // TODO: handle error return -1; } @@ -95,8 +95,8 @@ int pgFileOpen(SPgFile **ppPgFile, const char *fname, TENV *pEnv) { int pgFileClose(SPgFile *pPgFile) { if (pPgFile) { - if (pPgFile->fd >= 0) { - close(pPgFile->fd); + if (pPgFile->pFile != NULL) { + taosCloseFile(&pPgFile->pFile); } tfree(pPgFile->fname); @@ -201,7 +201,7 @@ static int pgFileRead(SPgFile *pPgFile, pgno_t pgno, uint8_t *pData) { pTData = pData; szToRead = pgSize; for (; szToRead > 0;) { - rsize = pread(pPgFile->fd, pTData, szToRead, pgno * pgSize); + rsize = pread(pPgFile->pFile, pTData, szToRead, pgno * pgSize); if (rsize < 0) { if (errno == EINTR) { continue; diff --git a/source/libs/tdb/src/inc/tdbPgFile.h b/source/libs/tdb/src/inc/tdbPgFile.h index 2a7116a0dd232b1933c030e72a4e95166af063ef..eaeebf6b9d3d6260b181b0e8fd9607a2befe7482 100644 --- a/source/libs/tdb/src/inc/tdbPgFile.h +++ b/source/libs/tdb/src/inc/tdbPgFile.h @@ -38,7 +38,7 @@ struct SPgFile { uint8_t fileid[TDB_FILE_ID_LEN]; // file id pgno_t lsize; // page file logical size (for count) pgno_t fsize; // real file size on disk (for rollback) - int fd; + TdFilePtr pFile; SPgFileListNode envHash; SPgFileListNode envPgfList; }; diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 9002879b10ae6e03f0cb318e42d0ffed8b730aba..f686703643d4ba0aa420b3d6ff7284c352c32970 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -25,7 +25,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir); static STfsDisk *tfsNextDisk(STfs *pTfs, SDiskIter *pIter); STfs *tfsOpen(SDiskCfg *pCfg, int32_t ndisk) { - if (ndisk < 0 || ndisk > TFS_MAX_DISKS) { + if (ndisk <= 0 || ndisk > TFS_MAX_DISKS) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } diff --git a/source/libs/tfs/test/tfsTest.cpp b/source/libs/tfs/test/tfsTest.cpp index 178d115c59c8aeae010fc98d60da777304c70fa2..af66304f84732715dbde1b4f69119f1b4a23080e 100644 --- a/source/libs/tfs/test/tfsTest.cpp +++ b/source/libs/tfs/test/tfsTest.cpp @@ -230,10 +230,11 @@ TEST_F(TfsTest, 04_File) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); @@ -638,10 +639,11 @@ TEST_F(TfsTest, 05_MultiDisk) { EXPECT_EQ(tfsMkdir(pTfs, "t3"), 0); - FILE *fp = fopen(f1.aname, "w"); - ASSERT_NE(fp, nullptr); - fwrite("12345678", 1, 5, fp); - fclose(fp); + // FILE *fp = fopen(f1.aname, "w"); + TdFilePtr pFile = taosOpenFile(f1.aname, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + ASSERT_NE(pFile, nullptr); + taosWriteFile(pFile, "12345678", 5); + taosCloseFile(&pFile); char base[128] = {0}; tfsBasename(&f1, base); diff --git a/source/libs/transport/inc/rpcUdp.h b/source/libs/transport/inc/rpcUdp.h index c1da6a92402367d23677eb5eb3753782248fb46e..0c651d07ed4662305e107c09a7312f372aa98339 100644 --- a/source/libs/transport/inc/rpcUdp.h +++ b/source/libs/transport/inc/rpcUdp.h @@ -30,7 +30,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t void taosFreeMsgHdr(void *hdr); int taosMsgHdrSize(void *hdr); -void taosSendMsgHdr(void *hdr, int fd); +void taosSendMsgHdr(void *hdr, TdFilePtr pFile); void taosInitMsgHdr(void **hdr, void *dest, int maxPkts); void taosSetMsgHdrData(void *hdr, char *data, int dataLen); diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index a36b671eb459268c1fcf51ca80ac21a88d050e37..d080db753d525362ecd8cca464f17f6d80fbd19e 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -66,6 +66,7 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + bool (*pfp)(void* parent, tmsg_t msgType); int32_t refCount; void* parent; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 72c1ff689374873794265fcb83908aae41c571be..e1319da1624b0b32c9cd0e987768e0ea8e66e022 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -143,6 +143,7 @@ typedef struct SRpcConn { static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; + // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 @@ -222,7 +223,7 @@ static void rpcInitImp(void) { tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } -int32_t rpcInit(void) { +int32_t rpcInit() { pthread_once(&tsRpcInitOnce, rpcInitImp); return 0; } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 48c15ca286e6202be5d71ca7791272b0a07ce3f3..b45683617f6343f5df79fb85e95e44aac3a8c056 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -29,7 +29,12 @@ void* rpcOpen(const SRpcInit* pInit) { if (pInit->label) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1); } + + // register callback handle pRpc->cfp = pInit->cfp; + pRpc->afp = pInit->afp; + pRpc->pfp = pInit->pfp; + if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } else { @@ -107,7 +112,7 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } void rpcCancelRequest(int64_t rid) { return; } -int32_t rpcInit(void) { +int32_t rpcInit() { // impl later return 0; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 8312c0217c13d40a43b57c82dc22ff04d5ddcd65..fb76f38fe51711b17bdb940adf0d241143ebe936 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -127,15 +127,14 @@ static void clientHandleResp(SCliConn* conn) { // buf's mem alread translated to rpcMsg.pCont transClearBuffer(&conn->readBuf); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg = {0}; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; - if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || - rpcMsg.msgType == TDMT_VND_RES_READY_RSP) { + if (pRpc->pfp != NULL && (pRpc->pfp)(pRpc->parent, rpcMsg.msgType)) { rpcMsg.handle = conn; conn->persist = 1; tDebug("client conn %p persist by app", conn); @@ -185,18 +184,13 @@ static void clientHandleExcept(SCliConn* pConn) { clientConnDestroy(pConn, true); return; } - SCliMsg* pMsg = pConn->data; - - tmsg_t msgType = TDMT_MND_CONNECT; - if (pMsg != NULL) { - msgType = pMsg->msg.msgType; - } + SCliMsg* pMsg = pConn->data; STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - rpcMsg.msgType = msgType + 1; + rpcMsg.msgType = pMsg->msg.msgType + 1; if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); @@ -445,7 +439,7 @@ static void clientConnCb(uv_connect_t* req, int status) { addrlen = sizeof(pConn->locaddr); uv_tcp_getsockname((uv_tcp_t*)pConn->stream, (struct sockaddr*)&pConn->locaddr, &addrlen); - tTrace("client conn %p create", pConn); + tTrace("client conn %p connect to server successfully", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -524,6 +518,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect + tTrace("client conn %p try to connect to %s:%d", conn, pMsg->ctx->ip, pMsg->ctx->port); uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 9fca371bf30588f7bdd50f5a3cc452e000bd03eb..f0db054797b936b8fde9fa3b241c8dbff909bb90 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -413,11 +413,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } else { uvStartSendResp(msg); } - // uv_buf_t wb; - // uvPrepareSendData(msg, &wb); - // uv_timer_stop(conn->pTimer); - - // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } static void uvAcceptAsyncCb(uv_async_t* async) { @@ -490,7 +485,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -730,14 +724,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -774,19 +763,12 @@ void rpcSendResponse(const SRpcMsg* pMsg) { SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); srvMsg->pConn = pConn; srvMsg->msg = *pMsg; - - // pthread_mutex_lock(&pThrd->msgMtx); - // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); - // pthread_mutex_unlock(&pThrd->msgMtx); - tTrace("server conn %p start to send resp", pConn); transSendAsync(pThrd->asyncPool, &srvMsg->q); - // uv_async_send(pThrd->workerAsync); } int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { SSrvConn* pConn = thandle; - // struct sockaddr* pPeerName = &pConn->peername; struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); diff --git a/source/libs/transport/test/pushClient.c b/source/libs/transport/test/pushClient.c index f1aadafacc23f8bfd166589604789c88f7d0ecfd..4842a0c800b53a87f8f6713bb1fd045adaa9083c 100644 --- a/source/libs/transport/test/pushClient.c +++ b/source/libs/transport/test/pushClient.c @@ -192,7 +192,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/pushServer.c b/source/libs/transport/test/pushServer.c index 0bcc47383b7160b92c0c5efb10406576a8a63cb9..a1c181ac982fab873d6121b161930fc39ace5ad4 100644 --- a/source/libs/transport/test/pushServer.c +++ b/source/libs/transport/test/pushServer.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { @@ -181,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_APPEND | TD_FILE_CTEATE | TD_FILE_WRITE); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -190,8 +190,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cc6a63d3cd7277ced2dc6575dd8389b639bd124a..bcdf32bf6aaae04b4e526a9a6e09395f454d1be6 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rsclient.c b/source/libs/transport/test/rsclient.c index 65170d4abb2745a469dfda3e4146c2ea85405b33..26a02eb05b7b704e32025731238453ffc3daece9 100644 --- a/source/libs/transport/test/rsclient.c +++ b/source/libs/transport/test/rsclient.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) { rpcInit.label = "APP"; rpcInit.numOfThreads = 1; rpcInit.sessions = 100; - rpcInit.idleTime = tsShellActivityTimer*1000; + rpcInit.idleTime = 3000; //tsShellActivityTimer*1000; rpcInit.user = "michael"; rpcInit.secret = secret; rpcInit.ckey = "key"; @@ -148,7 +148,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index d1a587f4e5539a45001cafa28e58f134f9bcdad3..5432a07649e88c4852b3c1081d5a3837adc23598 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -22,7 +22,7 @@ int msgSize = 128; int commit = 0; -int dataFd = -1; +TdFilePtr pDataFile = NULL; STaosQueue *qhandle = NULL; STaosQset * qset = NULL; @@ -43,8 +43,8 @@ void processShellMsg() { for (int i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); - if (dataFd >= 0) { - if (write(dataFd, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { + if (pDataFile != NULL) { + if (taosWriteFile(pDataFile, pRpcMsg->pCont, pRpcMsg->contLen) < 0) { tInfo("failed to write data file, reason:%s", strerror(errno)); } } @@ -52,7 +52,7 @@ void processShellMsg() { if (commit >= 2) { num += numOfMsgs; - // if (taosFsync(dataFd) < 0) { + // if (taosFsync(pDataFile) < 0) { // tInfo("failed to flush data to file, reason:%s", strerror(errno)); //} @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { tsAsyncLog = 0; rpcInit.connType = TAOS_CONN_SERVER; - taosInitLog("server.log", 100000, 10); + taosInitLog("server.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { @@ -170,8 +170,8 @@ int main(int argc, char *argv[]) { tInfo("RPC server is running, ctrl-c to exit"); if (commit) { - dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); - if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); + pDataFile = taosOpenFile(dataName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pDataFile == NULL) tInfo("failed to open data file, reason:%s", strerror(errno)); } qhandle = taosOpenQueue(); qset = taosOpenQset(); @@ -179,8 +179,8 @@ int main(int argc, char *argv[]) { processShellMsg(); - if (dataFd >= 0) { - close(dataFd); + if (pDataFile != NULL) { + taosCloseFile(&pDataFile); remove(dataName); } diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index c5d7f5664a4665529e9068836a846518aa6abf2c..b7ef296b9deae24cb950ec5e20dfbe6083931bf6 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -170,7 +170,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("client.log", 100000, 10); + taosInitLog("client.log", 10); void *pRpc = rpcOpen(&rpcInit); if (pRpc == NULL) { diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc index d1fefe2c727f8544ddc401bfbc8ed147009ffc93..6db709da513b6f169d12ea9e8aa3f9292544d2a1 100644 --- a/source/libs/transport/test/transUT.cc +++ b/source/libs/transport/test/transUT.cc @@ -18,7 +18,7 @@ #include "tep.h" #include "tglobal.h" #include "trpc.h" -#include "ulog.h" +#include "tlog.h" using namespace std; const char *label = "APP"; @@ -148,7 +148,6 @@ class TransObj { wDebugFlag = 0; sDebugFlag = 0; tsdbDebugFlag = 0; - cqDebugFlag = 0; tscEmbeddedInUtil = 1; tsAsyncLog = 0; @@ -156,9 +155,8 @@ class TransObj { taosRemoveDir(path.c_str()); taosMkDir(path.c_str()); - char temp[PATH_MAX]; - snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str()); - if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1) != 0) { printf("failed to init log file\n"); } cli = new Client; diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 4624e05f101cb3c079a4e632f939d8beb9634a9e..ca0c7561412f9614530e85129f0604f5c3da5db7 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -16,7 +16,7 @@ #ifndef _TD_WAL_INT_H_ #define _TD_WAL_INT_H_ -#include "compare.h" +#include "tcompare.h" #include "taoserror.h" #include "tchecksum.h" #include "tcoding.h" diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 2d6fb8fc761aa0f69fb7d2d9ac7a2cb37c3ed740..ae0b0bd849ba6d21a315c9f8f38eca163c46776f 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -69,8 +69,8 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { int readSize = TMIN(WAL_MAX_SIZE + 2, statbuf.st_size); pLastFileInfo->fileSize = statbuf.st_size; - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -79,15 +79,15 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { char* buf = malloc(readSize + 5); if (buf == NULL) { - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; return -1; } - taosLSeekFile(fd, -readSize, SEEK_END); - if (readSize != taosReadFile(fd, buf, readSize)) { + taosLSeekFile(pFile, -readSize, SEEK_END); + if (readSize != taosReadFile(pFile, buf, readSize)) { free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -108,12 +108,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { // file has to be deleted free(buf); - taosCloseFile(fd); + taosCloseFile(&pFile); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } } - taosCloseFile(fd); + taosCloseFile(&pFile); SWalHead *lastEntry = (SWalHead*)found; return lastEntry->head.version; @@ -364,18 +364,18 @@ int walSaveMeta(SWal* pWal) { int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer + 1, fnameStr); - FileFd metaFd = taosOpenFileCreateWrite(fnameStr); - if (metaFd < 0) { + TdFilePtr pMataFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE); + if (pMataFile == NULL) { return -1; } char* serialized = walMetaSerialize(pWal); int len = strlen(serialized); - if (len != taosWriteFile(metaFd, serialized, len)) { + if (len != taosWriteFile(pMataFile, serialized, len)) { // TODO:clean file return -1; } - taosCloseFile(metaFd); + taosCloseFile(&pMataFile); // delete old file if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); @@ -404,20 +404,20 @@ int walLoadMeta(SWal* pWal) { return -1; } memset(buf, 0, size + 5); - FileFd fd = taosOpenFileRead(fnameStr); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pFile == NULL) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } - if (taosReadFile(fd, buf, size) != size) { + if (taosReadFile(pFile, buf, size) != size) { terrno = TAOS_SYSTEM_ERROR(errno); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return -1; } // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - taosCloseFile(fd); + taosCloseFile(&pFile); free(buf); return code; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d5c28d9d9bb6bf76300efeb7ced7d2d38c01272b..b1caeab3b1396ae601326217c956133d776d54f8 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -14,10 +14,9 @@ */ #define _DEFAULT_SOURCE -#include "compare.h" +#include "tcompare.h" #include "os.h" #include "taoserror.h" -#include "tfile.h" #include "tref.h" #include "walInt.h" @@ -40,15 +39,9 @@ int32_t walInit() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); if (old == 1) return 0; - int code = tfInit(); - if (code != 0) { - wError("failed to init tfile since %s", tstrerror(code)); - atomic_store_8(&tsWal.inited, 0); - return code; - } tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - code = walCreateThread(); + int32_t code = walCreateThread(); if (code != 0) { wError("failed to init wal module since %s", tstrerror(code)); atomic_store_8(&tsWal.inited, 0); @@ -89,8 +82,8 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { // open meta walResetVer(&pWal->vers); - pWal->writeLogTfd = -1; - pWal->writeIdxTfd = -1; + pWal->pWriteLogTFile = NULL; + pWal->pWriteIdxTFile = NULL; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); if (pWal->fileInfoSet == NULL) { @@ -164,10 +157,10 @@ int32_t walAlter(SWal *pWal, SWalCfg *pCfg) { void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); - tfClose(pWal->writeLogTfd); - pWal->writeLogTfd = -1; - tfClose(pWal->writeIdxTfd); - pWal->writeIdxTfd = -1; + taosCloseFile(&pWal->pWriteLogTFile); + pWal->pWriteLogTFile = NULL; + taosCloseFile(&pWal->pWriteIdxTFile); + pWal->pWriteIdxTFile = NULL; walSaveMeta(pWal); taosArrayDestroy(pWal->fileInfoSet); pWal->fileInfoSet = NULL; @@ -207,7 +200,7 @@ static void walFsyncAll() { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); - int32_t code = tfFsync(pWal->writeLogTfd); + int32_t code = taosFsyncFile(pWal->pWriteLogTFile); if (code != 0) { wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1d9201f69dba28e92d86379355f6dcc3acb36606..9e1ffeae7f83deb4692ba221ad3f334fcc6965a2 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include "tfile.h" #include "walInt.h" #include "taoserror.h" @@ -25,8 +24,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } pRead->pWal = pWal; - pRead->readIdxTfd = -1; - pRead->readLogTfd = -1; + pRead->pReadIdxTFile = NULL; + pRead->pReadLogTFile = NULL; pRead->curVersion = -1; pRead->curFileFirstVer = -1; pRead->capacity = 0; @@ -41,8 +40,8 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) { } void walCloseReadHandle(SWalReadHandle *pRead) { - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); tfree(pRead->pHead); free(pRead); } @@ -52,24 +51,24 @@ int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; - int64_t idxTfd = pRead->readIdxTfd; - int64_t logTfd = pRead->readLogTfd; + TdFilePtr pIdxTFile = pRead->pReadIdxTFile; + TdFilePtr pLogTFile = pRead->pReadLogTFile; // seek position int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); - code = tfLseek(idxTfd, offset, SEEK_SET); + code = taosLSeekFile(pIdxTFile, offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } // TODO:deserialize ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -80,24 +79,24 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { char fnameStr[WAL_FILE_LEN]; - tfClose(pRead->readIdxTfd); - tfClose(pRead->readLogTfd); + taosCloseFile(&pRead->pReadIdxTFile); + taosCloseFile(&pRead->pReadLogTFile); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); - int64_t logTfd = tfOpenRead(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); - int64_t idxTfd = tfOpenRead(fnameStr); - if (idxTfd < 0) { + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); + if (pIdxTFile == NULL) { return -1; } - pRead->readLogTfd = logTfd; - pRead->readIdxTfd = idxTfd; + pRead->pReadLogTFile = pLogTFile; + pRead->pReadIdxTFile = pIdxTFile; return 0; } @@ -145,9 +144,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } } - if (!tfValid(pRead->readLogTfd)) return -1; + if (!taosValidFile(pRead->pReadLogTFile)) return -1; - code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); + code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead)); if (code != sizeof(SWalHead)) { return -1; } @@ -165,7 +164,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != taosReadFile(pRead->pReadLogTFile, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } @@ -202,7 +201,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { } *ppHead = ptr; } - if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->pWriteLogTFile, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } // TODO: endian compatibility processing after read @@ -215,7 +214,7 @@ int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { *ppHead = NULL; return -1; } - if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->pWriteLogTFile, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } // TODO: endian compatibility processing after read diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 91b172444b53c409b4d4db35a81ea80dc136f21a..140b7ddc32614662ceb86daae8f3beafb1ff8926 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -16,32 +16,31 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tfile.h" #include "tref.h" #include "walInt.h" static int walSeekWritePos(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd = pWal->writeIdxTfd; - int64_t logTfd = pWal->writeLogTfd; + TdFilePtr pIdxTFile = pWal->pWriteIdxTFile; + TdFilePtr pLogTFile = pWal->pWriteLogTFile; // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } SWalIdxEntry entry; // TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + code = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } ASSERT(entry.ver == ver); - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -50,43 +49,43 @@ static int walSeekWritePos(SWal* pWal, int64_t ver) { } int walSetWrite(SWal* pWal) { - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; return 0; } int walChangeWrite(SWal* pWal, int64_t ver) { int code = 0; - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; char fnameStr[WAL_FILE_LEN]; - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } } - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -103,23 +102,23 @@ int walChangeWrite(SWal* pWal, int64_t ver) { int64_t fileFirstVer = pFileInfo->firstVer; walBuildIdxName(pWal, fileFirstVer, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeIdxTfd = -1; + pWal->pWriteIdxTFile = NULL; return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { - tfClose(idxTfd); + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { + taosCloseFile(&pIdxTFile); terrno = TAOS_SYSTEM_ERROR(errno); - pWal->writeLogTfd = -1; + pWal->pWriteLogTFile = NULL; return -1; } - pWal->writeLogTfd = logTfd; - pWal->writeIdxTfd = idxTfd; + pWal->pWriteLogTFile = pLogTFile; + pWal->pWriteIdxTFile = pIdxTFile; pWal->writeCur = idx; return fileFirstVer; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index a4b34dee3752a2e57db48d523d6a1e2a1c7fcf77..4b1f0ba30637a01a78a01f9db5a4fd533e1c87dd 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,7 +18,6 @@ #include "os.h" #include "taoserror.h" #include "tchecksum.h" -#include "tfile.h" #include "walInt.h" int32_t walCommit(SWal *pWal, int64_t ver) { @@ -65,15 +64,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t idxTfd = tfOpenReadWrite(fnameStr); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); // TODO:change to deserialize function - if (idxTfd < 0) { + if (pIdxTFile == NULL) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); - code = tfLseek(idxTfd, idxOff, SEEK_SET); + code = taosLSeekFile(pIdxTFile, idxOff, SEEK_SET); if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; @@ -81,20 +80,20 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // read idx file and get log file pos // TODO:change to deserialize function SWalIdxEntry entry; - if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { + if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - int64_t logTfd = tfOpenReadWrite(fnameStr); - if (logTfd < 0) { + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + if (pLogTFile == NULL) { // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - code = tfLseek(logTfd, entry.offset, SEEK_SET); + code = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET); if (code < 0) { // TODO pthread_mutex_unlock(&pWal->mutex); @@ -102,8 +101,8 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } // validate offset SWalHead head; - ASSERT(tfValid(logTfd)); - int size = tfRead(logTfd, &head, sizeof(SWalHead)); + ASSERT(taosValidFile(pLogTFile)); + int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -118,11 +117,11 @@ int32_t walRollback(SWal *pWal, int64_t ver) { return -1; } // truncate old files - code = tfFtruncate(logTfd, entry.offset); + code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { return -1; } - code = tfFtruncate(idxTfd, idxOff); + code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { return -1; } @@ -203,31 +202,31 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if (pWal->writeIdxTfd != -1) { - code = tfClose(pWal->writeIdxTfd); + if (pWal->pWriteIdxTFile != NULL) { + code = taosCloseFile(&pWal->pWriteIdxTFile); if (code != 0) { return -1; } } - if (pWal->writeLogTfd != -1) { - code = tfClose(pWal->writeLogTfd); + if (pWal->pWriteLogTFile != NULL) { + code = taosCloseFile(&pWal->pWriteLogTFile); if (code != 0) { return -1; } } - int64_t idxTfd, logTfd; + TdFilePtr pIdxTFile, pLogTFile; // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); - idxTfd = tfOpenCreateWriteAppend(fnameStr); - if (idxTfd < 0) { + pIdxTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pIdxTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); - logTfd = tfOpenCreateWriteAppend(fnameStr); - if (logTfd < 0) { + pLogTFile = taosOpenFile(fnameStr, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } @@ -237,8 +236,8 @@ int walRoll(SWal *pWal) { } // switch file - pWal->writeIdxTfd = idxTfd; - pWal->writeLogTfd = logTfd; + pWal->pWriteIdxTFile = pIdxTFile; + pWal->pWriteLogTFile = pLogTFile; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; ASSERT(pWal->writeCur >= 0); @@ -248,7 +247,7 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + int size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); // TODO truncate @@ -282,16 +281,16 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in // must truncate explicitly first return -1; } - /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ + /*if (!tfValid(pWal->pWriteLogTFile)) return -1;*/ ASSERT(pWal->writeCur >= 0); pthread_mutex_lock(&pWal->mutex); - if (pWal->writeIdxTfd == -1 || pWal->writeLogTfd == -1) { + if (pWal->pWriteIdxTFile == NULL || pWal->pWriteLogTFile == NULL) { walSetWrite(pWal); - tfLseek(pWal->writeLogTfd, 0, SEEK_END); - tfLseek(pWal->writeIdxTfd, 0, SEEK_END); + taosLSeekFile(pWal->pWriteLogTFile, 0, SEEK_END); + taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); } pWal->writeHead.head.version = index; @@ -302,14 +301,14 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { // ftruncate code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -336,7 +335,7 @@ int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, in void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); - if (tfFsync(pWal->writeLogTfd) < 0) { + if (taosFsyncFile(pWal->pWriteLogTFile) < 0) { wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 8999646f6a7ad8b61b53f51e67d60610c0ee2028..c464073e5faeb6a9a08da886268f469189b2fbf1 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -71,7 +71,7 @@ int32_t taosMkDir(const char *dirname) { return code; } -void taosRemoveOldFiles(char *dirname, int32_t keepDays) { +void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { DIR *dir = opendir(dirname); if (dir == NULL) return; @@ -114,7 +114,7 @@ void taosRemoveOldFiles(char *dirname, int32_t keepDays) { rmdir(dirname); } -int32_t taosExpandDir(char *dirname, char *outname, int32_t maxlen) { +int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { wordexp_t full_path; if (0 != wordexp(dirname, &full_path, 0)) { //printf("failed to expand path:%s since %s", dirname, strerror(errno)); diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 7bc5e0a7fcfa759444d0d6d445a9b2eb1175f8e1..4c368fe895a4b395f8016f47d68795aee8eb2db6 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -13,54 +13,89 @@ * along with this program. If not, see . */ - #define _DEFAULT_SOURCE -#include "os.h" - -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#include "osEnv.h" -char tsOsName[10] = "Windows"; -char configDir[PATH_MAX] = "C:/TDengine/cfg"; -char tsDataDir[PATH_MAX] = "C:/TDengine/data"; -char tsLogDir[PATH_MAX] = "C:/TDengine/log"; -char tsScriptDir[PATH_MAX] = "C:/TDengine/script"; -char tsTempDir[PATH_MAX] = "C:\\Windows\\Temp"; +extern void taosWinSocketInit(); -extern taosWinSocketInit(); +char configDir[PATH_MAX] = {0}; +char tsDataDir[PATH_MAX] = {0}; +char tsLogDir[PATH_MAX] = {0}; +char tsTempDir[PATH_MAX] = {0}; +SDiskSpace tsDataSpace = {0}; +SDiskSpace tsLogSpace = {0}; +SDiskSpace tsTempSpace = {0}; +char tsOsName[16] = {0}; +char tsTimezone[TD_TIMEZONE_LEN] = {0}; +char tsLocale[TD_LOCALE_LEN] = {0}; +char tsCharset[TD_CHARSET_LEN] = {0}; +int8_t tsDaylight = 0; +bool tsEnableCoreFile = 0; +int64_t tsPageSize = 0; +int64_t tsOpenMax = 0; +int64_t tsStreamMax = 0; +int32_t tsNumOfCores = 0; +int32_t tsTotalMemoryMB = 0; void osInit() { + srand(taosSafeRand()); + taosGetSystemLocale(tsLocale, tsCharset); + taosGetSystemTimezone(tsTimezone); + taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); + taosGetSystemInfo(); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) taosWinSocketInit(); const char *tmpDir = getenv("tmp"); if (tmpDir == NULL) { tmpDir = getenv("temp"); } - if (tmpDir != NULL) { strcpy(tsTempDir, tmpDir); } -} - -#elif defined(_TD_DARWIN_64) -char tsOsName[10] = "Darwin"; -char configDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsDataDir[PATH_MAX] = "/usr/local/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/usr/local/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/usr/local/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/taosd"; + if (configDir[0] == 0) { + strcpy(configDir, "C:\\TDengine\\cfg"); + } + strcpy(tsDataDir, "C:\\TDengine\\data"); + strcpy(tsLogDir, "C:\\TDengine\\log"); + strcpy(tsTempDir, "C:\\Windows\\Temp"); + strcpy(tsOsName, "Windows"); -void osInit() {} +#elif defined(_TD_DARWIN_64) + if (configDir[0] == 0) { + strcpy(configDir, "/tmp/taosd"); + } + strcpy(tsDataDir, "/usr/local/var/lib/taos"); + strcpy(tsLogDir, "/usr/local/var/log/taos"); + strcpy(tsTempDir, "/usr/local/etc/taos"); + strcpy(tsOsName, "Darwin"); #else + if (configDir[0] == 0) { + strcpy(configDir, "/etc/taos"); + } + strcpy(tsDataDir, "/var/lib/taos"); + strcpy(tsLogDir, "/var/log/taos"); + strcpy(tsTempDir, "/tmp"); + strcpy(tsOsName, "Linux"); + +#endif +} -char tsOsName[10] = "Linux"; -char configDir[PATH_MAX] = "/etc/taos"; -char tsDataDir[PATH_MAX] = "/var/lib/taos"; -char tsLogDir[PATH_MAX] = "/var/log/taos"; -char tsScriptDir[PATH_MAX] = "/etc/taos"; -char tsTempDir[PATH_MAX] = "/tmp/"; +void osUpdate() { + if (tsLogDir[0] != 0) { + taosGetDiskSize(tsLogDir, &tsLogSpace.size); + } + if (tsDataDir[0] != 0) { + taosGetDiskSize(tsDataDir, &tsDataSpace.size); + } + if (tsTempDir[0] != 0) { + taosGetDiskSize(tsTempDir, &tsTempSpace.size); + } +} -void osInit() {} +bool osLogSpaceAvailable() { return tsLogSpace.reserved <= tsLogSpace.size.avail; } -#endif +void osSetTimezone(const char *timezone) { taosSetSystemTimezone(tsTimezone, tsTimezone, &tsDaylight); } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 5a67ab73eb4f0d465ab219bca8ec3ee6ed78169e..fbb0e752571afe03467954e56a8984331912173d 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -12,9 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#define ALLOW_FORBID_FUNC #include "os.h" +#define MAX_FPRINTFLINE_BUFFER_SIZE (1000) + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include @@ -38,14 +40,19 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #include #include #include +#define LINUX_FILE_NO_TEXT_OPTION 0 +#define O_TEXT LINUX_FILE_NO_TEXT_OPTION #endif -void taosCloseFile(FileFd fd) { - close(fd); - fd = FD_INITIALIZER; -} +typedef int32_t FileFd; -void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, char *dstPath) { +typedef struct TdFile { + int refId; + FileFd fd; + FILE *fp; +} * TdFilePtr, TdFile; + +void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; @@ -72,7 +79,7 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch const char *tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; int32_t len = strlen(inputTmpDir); memcpy(tmpPath, inputTmpDir, len); static uint64_t seqId = 0; @@ -96,13 +103,181 @@ void taosGetTmpfilePath(const char * inputTmpDir, const char *fileNamePrefix, ch #endif } -int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { +int64_t taosCopyFile(const char *from, const char *to) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + char buffer[4096]; + int64_t size = 0; + int64_t bytes; + + // fidfrom = open(from, O_RDONLY); + TdFilePtr pFileFrom = taosOpenFile(from, TD_FILE_READ); + if (pFileFrom == NULL) goto _err; + + // fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); + TdFilePtr pFileTo = taosOpenFile(to, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_EXCL); + if (pFileTo == NULL) goto _err; + + while (true) { + bytes = taosReadFile(pFileFrom, buffer, sizeof(buffer)); + if (bytes < 0) goto _err; + if (bytes == 0) break; + + size += bytes; + + if (taosWriteFile(pFileTo, (void *)buffer, bytes) < bytes) goto _err; + if (bytes < sizeof(buffer)) break; + } + + taosFsyncFile(pFileTo); + + taosCloseFile(&pFileFrom); + taosCloseFile(&pFileTo); + return size; + +_err: + if (pFileFrom != NULL) taosCloseFile(&pFileFrom); + if (pFileTo != NULL) taosCloseFile(&pFileTo); + remove(to); + return -1; +#endif +} + +int32_t taosRenameFile(const char *oldName, const char *newName) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); + if (code < 0) { + // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#else + int32_t code = rename(oldName, newName); + if (code < 0) { + // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + } + + return code; +#endif +} + +int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + struct stat fileStat; + int32_t code = stat(path, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +void autoDelFileListAdd(const char *path) { return; } + +TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return NULL; +#else + int fd = -1; + FILE *fp = NULL; + if (tdFileOptions & TD_FILE_STREAM) { + char *mode = NULL; + if (tdFileOptions & TD_FILE_APPEND) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "at+" : "ab+"; + }else if (tdFileOptions & TD_FILE_TRUNC) { + mode = (tdFileOptions & TD_FILE_TEXT) ? "wt+" : "wb+"; + }else { + mode = (tdFileOptions & TD_FILE_TEXT) ? "rt+" : "rb+"; + } + assert(!(tdFileOptions & TD_FILE_EXCL)); + fp = fopen(path, mode); + if (fp == NULL) { + return NULL; + } + } else { + int access = O_BINARY; + access |= (tdFileOptions & TD_FILE_CTEATE) ? O_CREAT : 0; + if ((tdFileOptions & TD_FILE_WRITE) && (tdFileOptions & TD_FILE_READ)) { + access |= O_RDWR; + } else if (tdFileOptions & TD_FILE_WRITE) { + access |= O_WRONLY; + } else if (tdFileOptions & TD_FILE_READ) { + access |= O_RDONLY; + } + access |= (tdFileOptions & TD_FILE_TRUNC) ? O_TRUNC : 0; + access |= (tdFileOptions & TD_FILE_APPEND) ? O_APPEND : 0; + access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0; + access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0; + fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO); + if (fd == -1) { + return NULL; + } + } + + if (tdFileOptions & TD_FILE_AUTO_DEL) { + autoDelFileListAdd(path); + } + + TdFilePtr pFile = (TdFilePtr)malloc(sizeof(TdFile)); + if (pFile == NULL) { + if (fd >= 0) close(fd); + if (fp != NULL) fclose(fp); + return NULL; + } + pFile->fd = fd; + pFile->fp = fp; + pFile->refId = 0; + return pFile; +#endif +} + +int64_t taosCloseFile(TdFilePtr *ppFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if (ppFile == NULL || *ppFile == NULL || (*ppFile)->fd == -1) { + return 0; + } + if ((*ppFile)->fp != NULL) { + fflush((*ppFile)->fp); + fclose((*ppFile)->fp); + (*ppFile)->fp = NULL; + } + if ((*ppFile)->fd >= 0) { + fsync((*ppFile)->fd); + close((*ppFile)->fd); + (*ppFile)->fd = -1; + } + (*ppFile)->refId = 0; + free(*ppFile); + *ppFile = NULL; + return 0; +#endif +} + +int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); int64_t leftbytes = count; int64_t readbytes; - char * tbuf = (char *)buf; + char *tbuf = (char *)buf; while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, (uint32_t)leftbytes); + readbytes = read(pFile->fd, (void *)tbuf, (uint32_t)leftbytes); if (readbytes < 0) { if (errno == EINTR) { continue; @@ -120,13 +295,26 @@ int64_t taosReadFile(FileFd fd, void *buf, int64_t count) { return count; } -int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { - int64_t nleft = n; +int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + return pread(pFile->fd, buf, count, offset); +} + +int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + + int64_t nleft = count; int64_t nwritten = 0; - char * tbuf = (char *)buf; + char *tbuf = (char *)buf; while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, (uint32_t)nleft); + nwritten = write(pFile->fd, (void *)tbuf, (uint32_t)nleft); if (nwritten < 0) { if (errno == EINTR) { continue; @@ -136,49 +324,151 @@ int64_t taosWriteFile(FileFd fd, const void *buf, int64_t n) { nleft -= nwritten; tbuf += nwritten; } + fsync(pFile->fd); + return count; +} - return n; +int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + return (int64_t)lseek(pFile->fd, (long)offset, whence); } -int64_t taosLSeekFile(FileFd fd, int64_t offset, int32_t whence) { return (int64_t)lseek(fd, (long)offset, whence); } +int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); -int64_t taosCopyFile(const char *from, const char *to) { + struct stat fileStat; + int32_t code = fstat(pFile->fd, &fileStat); + if (code < 0) { + return code; + } + + if (size != NULL) { + *size = fileStat.st_size; + } + + if (mtime != NULL) { + *mtime = fileStat.st_mtime; + } + + return 0; +#endif +} + +int32_t taosLockFile(TdFilePtr pFile) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - char buffer[4096]; - int fidto = -1, fidfrom = -1; - int64_t size = 0; - int64_t bytes; + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); - fidfrom = open(from, O_RDONLY); - if (fidfrom < 0) goto _err; + return (int32_t)flock(pFile->fd, LOCK_EX | LOCK_NB); +#endif +} - fidto = open(to, O_WRONLY | O_CREAT | O_EXCL, 0755); - if (fidto < 0) goto _err; +int32_t taosUnLockFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return 0; +#else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); - while (true) { - bytes = taosReadFile(fidfrom, buffer, sizeof(buffer)); - if (bytes < 0) goto _err; - if (bytes == 0) break; + return (int32_t)flock(pFile->fd, LOCK_UN | LOCK_NB); +#endif +} - size += bytes; +int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } - if (taosWriteFile(fidto, (void *)buffer, bytes) < bytes) goto _err; - if (bytes < sizeof(buffer)) break; + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + LARGE_INTEGER li_0; + li_0.QuadPart = (int64_t)0; + BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); + if (!cur) { + uError("SetFilePointerEx Error getting current position in file.\n"); + return -1; } - taosFsyncFile(fidto); + LARGE_INTEGER li_size; + li_size.QuadPart = l_size; + BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); + if (cur2 == 0) { + int error = GetLastError(); + uError("SetFilePointerEx GetLastError is: %d\n", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } - taosCloseFile(fidfrom); - taosCloseFile(fidto); - return size; + if (!SetEndOfFile(h)) { + int error = GetLastError(); + uError("SetEndOfFile GetLastError is:%d", error); + switch (error) { + case ERROR_INVALID_HANDLE: + errno = EBADF; + break; + default: + errno = EIO; + break; + } + return -1; + } -_err: - if (fidfrom >= 0) taosCloseFile(fidfrom); - if (fidto >= 0) taosCloseFile(fidto); - remove(to); - return -1; + return 0; +#else + if (pFile == NULL) { + return 0; + } + assert(pFile->fd >= 0); + + return ftruncate(pFile->fd, l_size); +#endif +} + +int32_t taosFsyncFile(TdFilePtr pFile) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + if (pFile->fd < 0) { + errno = EBADF; + uError("%s\n", "fd arg was negative"); + return -1; + } + + HANDLE h = (HANDLE)_get_osfhandle(pFile->fd); + + return FlushFileBuffers(h); +#else + if (pFile == NULL) { + return 0; + } + + if (pFile->fp != NULL) return fflush(pFile->fp); + if (pFile->fp >= 0) return fsync(pFile->fd); + + return 0; #endif } @@ -261,7 +551,7 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = fread(buf, 1, n, in_file); @@ -286,7 +576,7 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { } off_t len = count; while (len > 0) { - char buf[1024 * 16]; + char buf[1024 * 16]; off_t n = sizeof(buf); if (len < n) n = len; size_t m = read(sfd, buf, n); @@ -301,12 +591,17 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { #else -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { +int64_t taosSendFile(SocketFd fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { + if (pFileSrc == NULL) { + return 0; + } + assert(pFileSrc->fd >= 0); + int64_t leftbytes = size; int64_t sentbytes; while (leftbytes > 0) { - sentbytes = sendfile(dfd, sfd, offset, leftbytes); + sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); if (sentbytes == -1) { if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { continue; @@ -323,215 +618,66 @@ int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t size) { return size; } -int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size) { - return taosSendFile(fileno(outfile), fileno(infile), offset, size); -} - -#endif - -int32_t taosFtruncateFile(FileFd fd, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; } + assert(pFileOut->fd >= 0); - HANDLE h = (HANDLE)_get_osfhandle(fd); - - LARGE_INTEGER li_0; - li_0.QuadPart = (int64_t)0; - BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); - if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); - return -1; - } - - LARGE_INTEGER li_size; - li_size.QuadPart = l_size; - BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); - if (cur2 == 0) { - int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - if (!SetEndOfFile(h)) { - int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); - switch (error) { - case ERROR_INVALID_HANDLE: - errno = EBADF; - break; - default: - errno = EIO; - break; - } - return -1; - } - - return 0; -#else - return ftruncate(fd, l_size); -#endif + return taosSendFile(pFileOut->fd, pFileIn, offset, size); } -int32_t taosFsyncFile(FileFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - if (fd < 0) { - errno = EBADF; - uError("%s\n", "fd arg was negative"); - return -1; - } - - HANDLE h = (HANDLE)_get_osfhandle(fd); - - return FlushFileBuffers(h); -#else - return fsync(fd); #endif -} -int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); +void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { + if (pFile == NULL) { + return; } + assert(pFile->fp != NULL); + + char buffer[MAX_FPRINTFLINE_BUFFER_SIZE] = {0}; + va_list ap; + va_start(ap, format); + vfprintf(pFile->fp, format, ap); + va_end(ap); + fflush(pFile->fp); +} - return code; -#else - int32_t code = rename(oldName, newName); - if (code < 0) { - //printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); +void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { + if (pFile == NULL) { + return NULL; } + assert(pFile->fd >= 0); - return code; -#endif + void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); + return ptr; } -int32_t taosLockFile(int32_t fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return (int32_t)flock(fd, LOCK_EX | LOCK_NB); -#endif -} +bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } -int32_t taosUnLockFile(int32_t fd) { +int32_t taosUmaskFile(int32_t maskVal) { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) return 0; #else - return (int32_t)flock(fd, LOCK_UN | LOCK_NB); + return umask(maskVal); #endif } -int32_t taosUmaskFile(int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return umask(val); -#endif -} - -int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = stat(path, &fileStat); - if (code < 0) { - return code; - } - - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; +int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } +int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict__ ptrBuf) { + if (pFile == NULL) { + return -1; } + assert(pFile->fp != NULL); - return 0; -#endif + size_t len = 0; + return getline(ptrBuf, &len, pFile->fp); } - -int32_t taosFStatFile(int32_t fd, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - struct stat fileStat; - int32_t code = fstat(fd, &fileStat); - if (code < 0) { - return code; +int32_t taosEOFFile(TdFilePtr pFile) { + if (pFile == NULL) { + return 0; } + assert(pFile->fp != NULL); - if (size != NULL) { - *size = fileStat.st_size; - } - - if (mtime != NULL) { - *mtime = fileStat.st_mtime; - } - - return 0; -#endif -} - -int32_t taosOpenFileWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWrite(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteTrunc(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -int32_t taosOpenFileCreateWriteAppend(const char *path) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileRead(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - -FileFd taosOpenFileReadWrite(const char *path) { - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return 0; -#else - return open(path, O_RDWR, S_IRWXU | S_IRWXG | S_IRWXO); -#endif -} - - + return feof(pFile->fp); +} \ No newline at end of file diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c new file mode 100644 index 0000000000000000000000000000000000000000..47546f7deb2ddaf03106fd63e6f82a716f23a7fd --- /dev/null +++ b/source/os/src/osLocale.c @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "osLocale.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + {"utf8", "UTF-8"}, + {"936", "CP936"}, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} + +/** + * In some Linux systems, setLocale(LC_CTYPE, "") may return NULL, in which case the launch of + * both the TDengine Server and the Client may be interrupted. + * + * In case that the setLocale failed to be executed, the right charset needs to be set. + */ +void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { + char *locale = setlocale(LC_CTYPE, inLocale); + + // default locale or user specified locale is not valid, abort launch + if (inLocale == NULL || strlen(inLocale) == 0) { + //printf("Invalid locale:%s, please set the valid locale in config file\n", inLocale); + } + + if (!taosValidateEncodec(inCharSet)) { + printf("Invalid charset:%s, please set the valid charset in config file", inCharSet); + exit(-1); + } +} + +void taosGetSystemLocale(char *outLocale, char *outCharset) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + } + strcpy(outCharset, "cp936"); + +#elif defined(_TD_DARWIN_64) + /* + * originally from src/os/src/detail/osSysinfo.c + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + /* if user does not specify the charset, extract it from locale */ + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_CHARSET_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#else + /* + * POSIX format locale string: + * (Language Strings)_(Country/Region Strings).(code_page) + * + * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, + * + * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. + * + * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page + * for libiconv that is employed to convert string in this system. This program will automatically use + * UTF-8 instead as the charset. + * + * In case of windows client, the locale string is not valid POSIX format, user needs to set the + * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is + * CP936, CP437 for English charset. + * + */ + char sep = '.'; + char *locale = NULL; + + locale = setlocale(LC_CTYPE, ""); + if (locale == NULL) { + // printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); + strcpy(outLocale, "en_US.UTF-8"); + } else { + tstrncpy(outLocale, locale, TD_LOCALE_LEN); + // printf("locale not configured, set to system default:%s", outLocale); + } + + // if user does not specify the charset, extract it from locale + char *str = strrchr(outLocale, sep); + if (str != NULL) { + str++; + + char *revisedCharset = taosCharsetReplace(str); + tstrncpy(outCharset, revisedCharset, TD_LOCALE_LEN); + + free(revisedCharset); + // printf("charset not configured, set to system default:%s", outCharset); + } else { + strcpy(outCharset, "UTF-8"); + // printf("can't get locale and charset from system, set it to UTF-8"); + } + +#endif +} diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 973323a346b4d49179abf88a5c3f1950e688d8ad..b81e41b3cf3b9631ee778a233f9d12cddfe534ed 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -24,18 +24,18 @@ uint32_t taosRand(void) { return rand(); } uint32_t taosSafeRand(void) { - int fd; + TdFilePtr pFile; int seed; - fd = open("/dev/urandom", 0); - if (fd < 0) { + pFile = taosOpenFile("/dev/urandom", TD_FILE_READ); + if (pFile == NULL) { seed = (int)time(0); } else { - int len = read(fd, &seed, sizeof(seed)); + int len = taosReadFile(pFile, &seed, sizeof(seed)); if (len < 0) { seed = (int)time(0); } - close(fd); + taosCloseFile(&pFile); } return (uint32_t)seed; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e12f9493b3159e85593f1723ab018ff047614851..07d30276b7ba28205425c84634efd0bd6eeab4ae 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) @@ -150,7 +151,7 @@ int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #if !((defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) && defined(_MSC_VER)) -uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } +uint32_t taosInetAddr(const char *ipAddr) { return inet_addr(ipAddr); } const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } @@ -240,7 +241,7 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op #ifdef _MSC_VER //#if _MSC_VER >= 1900 -uint32_t taosInetAddr(char *ipAddr) { +uint32_t taosInetAddr(const char *ipAddr) { uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { diff --git a/source/os/src/osString.c b/source/os/src/osString.c index f714fd0dc4d26cf9f5c1391b34c6277b971cd74d..88ea4b3e1521103b4827c7b5646ec2ed92b556e6 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -18,24 +18,6 @@ #include #include -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} int64_t taosStr2int64(const char *str) { char *endptr = NULL; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index cae1b18b3c3dc876d9a7b2b54b1e4735ec417e4c..cf9c557f5ef5b36a23de42adf9b1a54e75a27f4c 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -16,15 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" -int32_t tsTotalMemoryMB = 0; -int64_t tsPageSize = 0; -int64_t tsOpenMax = 0; -int64_t tsStreamMax = 0; -int32_t tsNumOfCores = 1; -char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; -char tsLocale[TSDB_LOCALE_LEN] = {0}; -char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string - #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) /* @@ -89,25 +80,6 @@ bool taosGetProcMemory(float *memoryUsedMB) { return true; } -static void taosGetSystemTimezone() { - // get and set default timezone - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } else { - strcpy(tsTimezone, tz); - } -} - -static void taosGetSystemLocale() { - // get and set default locale - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - } - - strcpy(tsCharset, "cp936"); -} int32_t taosGetCpuCores() { SYSTEM_INFO info; @@ -200,8 +172,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -273,103 +243,6 @@ char *taosGetCmdlineByPID(int pid) { return ""; } #include #include -static void taosGetSystemTimezone() { - /* load time zone string from /etc/localtime */ - char buf[4096]; - char *tz = NULL; - { - int n = readlink("/etc/localtime", buf, sizeof(buf)); - if (n < 0) { - //printf("read /etc/localtime error, reason:%s", strerror(errno)); - return; - } - buf[n] = '\0'; - for (int i = n - 1; i >= 0; --i) { - if (buf[i] == '/') { - if (tz) { - tz = buf + i + 1; - break; - } - tz = buf + i + 1; - } - } - if (!tz || 0 == strchr(tz, '/')) { - //printf("parsing /etc/localtime failed"); - return; - } - - setenv("TZ", tz, 1); - tzset(); - } - - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], - -timezone / 3600); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * originally from src/os/src/detail/osSysinfo.c - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - /* if user does not specify the charset, extract it from locale */ - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} void taosKillSystem() { //printf("function taosKillSystem, exit!"); @@ -386,9 +259,6 @@ void taosGetSystemInfo() { long page_size = sysconf(_SC_PAGESIZE); tsTotalMemoryMB = physical_pages * page_size / (1024 * 1024); tsPageSize = page_size; - - taosGetSystemTimezone(); - taosGetSystemLocale(); } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { @@ -533,19 +403,17 @@ bool taosGetSysMemory(float *memoryUsedMB) { } bool taosGetProcMemory(float *memoryUsedMB) { - FILE *fp = fopen(tsProcMemFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcMemFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcMemFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { //printf("open file:%s failed", tsProcMemFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -556,7 +424,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { if (line == NULL) { //printf("read file:%s failed", tsProcMemFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -565,24 +433,24 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { - FILE *fp = fopen(tsSysCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { //printf("open file:%s failed", tsSysCpuFile); return false; } - size_t len; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsSysCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -590,24 +458,24 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { - FILE *fp = fopen(tsProcCpuFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcCpuFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { //printf("open file:%s failed", tsProcCpuFile); return false; } - size_t len = 0; char * line = NULL; - ssize_t _bytes = getline(&line, &len, fp); + ssize_t _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { //printf("read file:%s failed", tsProcCpuFile); - fclose(fp); + taosCloseFile(&pFile); return false; } @@ -620,115 +488,11 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } -static void taosGetSystemTimezone() { - /* - * NOTE: do not remove it. - * Enforce set the correct daylight saving time(DST) flag according - * to current time - */ - time_t tx1 = time(NULL); - struct tm tm1; - localtime_r(&tx1, &tm1); - - /* load time zone string from /etc/timezone */ - FILE *f = fopen("/etc/timezone", "r"); - char buf[68] = {0}; - if (f != NULL) { - int len = fread(buf, 64, 1, f); - if (len < 64 && ferror(f)) { - fclose(f); - //printf("read /etc/timezone error, reason:%s", strerror(errno)); - return; - } - - fclose(f); - - buf[sizeof(buf) - 1] = 0; - char *lineEnd = strstr(buf, "\n"); - if (lineEnd != NULL) { - *lineEnd = 0; - } - - // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables - if (strlen(buf) > 0) { - setenv("TZ", buf, 1); - } - } - // get and set default timezone - tzset(); - - /* - * get CURRENT time zone. - * system current time zone is affected by daylight saving time(DST) - * - * e.g., the local time zone of London in DST is GMT+01:00, - * otherwise is GMT+00:00 - */ - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; - tz += daylight; - - /* - * format example: - * - * Asia/Shanghai (CST, +0800) - * Europe/London (BST, +0100) - */ - snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); - - // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - //printf("timezone not configured, set to system default:%s", tsTimezone); -} - -/* - * POSIX format locale string: - * (Language Strings)_(Country/Region Strings).(code_page) - * - * example: en_US.UTF-8, zh_CN.GB18030, zh_CN.UTF-8, - * - * if user does not specify the locale in taos.cfg the program use default LC_CTYPE as system locale. - * - * In case of some CentOS systems, their default locale is "en_US.utf8", which is not valid code_page - * for libiconv that is employed to convert string in this system. This program will automatically use - * UTF-8 instead as the charset. - * - * In case of windows client, the locale string is not valid POSIX format, user needs to set the - * correct code_page for libiconv. Usually, the code_page of windows system with simple chinese is - * CP936, CP437 for English charset. - * - */ -static void taosGetSystemLocale() { // get and set default locale - char sep = '.'; - char *locale = NULL; - - locale = setlocale(LC_CTYPE, ""); - if (locale == NULL) { - //printf("can't get locale from system, set it to en_US.UTF-8 since error:%d:%s", errno, strerror(errno)); - strcpy(tsLocale, "en_US.UTF-8"); - } else { - tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN); - //printf("locale not configured, set to system default:%s", tsLocale); - } - - // if user does not specify the charset, extract it from locale - char *str = strrchr(tsLocale, sep); - if (str != NULL) { - str++; - - char *revisedCharset = taosCharsetReplace(str); - tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN); - - free(revisedCharset); - //printf("charset not configured, set to system default:%s", tsCharset); - } else { - strcpy(tsCharset, "UTF-8"); - //printf("can't get locale and charset from system, set it to UTF-8"); - } -} int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); } @@ -785,19 +549,17 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes = 0; - FILE *fp = fopen(tsSysNetFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsSysNetFile, "r"); + TdFilePtr pFile = taosOpenFile(tsSysNetFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { //printf("open file:%s failed", tsSysNetFile); return false; } ssize_t _bytes = 0; - size_t len = 2048; - char * line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + char * line = NULL; + while (!taosEOFFile(pFile)) { int64_t o_rbytes = 0; int64_t rpackts = 0; int64_t o_tbytes = 0; @@ -810,12 +572,12 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { int64_t nouse6 = 0; char nouse0[200] = {0}; - _bytes = getline(&line, &len, fp); + _bytes = taosGetLineFile(pFile, &line); if (_bytes < 0) { break; } - line[len - 1] = 0; + line[_bytes - 1] = 0; if (strstr(line, "lo:") != NULL) { continue; @@ -830,8 +592,8 @@ bool taosGetCardInfo(int64_t *bytes, int64_t *rbytes, int64_t *tbytes) { *bytes += (o_rbytes + o_tbytes); } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); return true; } @@ -873,22 +635,20 @@ bool taosGetBandSpeed(float *bandSpeedKb) { } bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { - FILE *fp = fopen(tsProcIOFile, "r"); - if (fp == NULL) { + // FILE *fp = fopen(tsProcIOFile, "r"); + TdFilePtr pFile = taosOpenFile(tsProcIOFile, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { //printf("open file:%s failed", tsProcIOFile); return false; } ssize_t _bytes = 0; - size_t len; char * line = NULL; char tmp[10]; int readIndex = 0; - while (!feof(fp)) { - tfree(line); - len = 0; - _bytes = getline(&line, &len, fp); + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); if ((_bytes < 0) || (line == NULL)) { break; } @@ -904,8 +664,8 @@ bool taosReadProcIO(int64_t *rchars, int64_t *wchars) { if (readIndex >= 2) break; } - tfree(line); - fclose(fp); + if(line != NULL) tfree(line); + taosCloseFile(&pFile); if (readIndex < 2) { //printf("read file:%s failed", tsProcIOFile); @@ -957,8 +717,6 @@ void taosGetSystemInfo() { taosGetCpuUsage(&tmp1, &tmp2); taosGetProcIO(&tmp1, &tmp2); - taosGetSystemTimezone(); - taosGetSystemLocale(); } void taosKillSystem() { @@ -1070,15 +828,15 @@ void taosSetCoreDump(bool enable) { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { - int fd; int len = 0; - fd = open("/proc/sys/kernel/random/uuid", 0); - if (fd < 0) { + // fd = open("/proc/sys/kernel/random/uuid", 0); + TdFilePtr pFile = taosOpenFile("/proc/sys/kernel/random/uuid", TD_FILE_READ); + if (pFile == NULL) { return -1; } else { - len = read(fd, uid, uidlen); - close(fd); + len = taosReadFile(pFile, uid, uidlen); + taosCloseFile(&pFile); } if (len >= 36) { @@ -1093,16 +851,17 @@ char *taosGetCmdlineByPID(int pid) { static char cmdline[1024]; sprintf(cmdline, "/proc/%d/cmdline", pid); - int fd = open(cmdline, O_RDONLY); - if (fd >= 0) { - int n = read(fd, cmdline, sizeof(cmdline) - 1); + // int fd = open(cmdline, O_RDONLY); + TdFilePtr pFile = taosOpenFile(cmdline, TD_FILE_READ); + if (pFile != NULL) { + int n = taosReadFile(pFile, cmdline, sizeof(cmdline) - 1); if (n < 0) n = 0; if (n > 0 && cmdline[n - 1] == '\n') --n; cmdline[n] = 0; - close(fd); + taosCloseFile(&pFile); } else { cmdline[0] = 0; } diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c new file mode 100644 index 0000000000000000000000000000000000000000..68da2ce25e1da8058dfeb1de60c6fb25c4f80ea3 --- /dev/null +++ b/source/os/src/osTimezone.c @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#if (_WIN64) +#include +#include +#include +#include +#include +#include +#pragma comment(lib, "Mswsock.lib ") +#endif +#include +#pragma warning(push) +#pragma warning(disable : 4091) +#include +#pragma warning(pop) +#elif defined(_TD_DARWIN_64) +#include +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +void taosSetSystemTimezone(const char *inTimezone, char *outTimezone, int8_t *outDaylight) { + if (inTimezone == NULL || inTimezone[0] == 0) return; + +#ifdef WINDOWS + char winStr[TD_LOCALE_LEN * 2]; + sprintf(winStr, "TZ=%s", inTimezone); + putenv(winStr); +#else + setenv("TZ", inTimezone, 1); +#endif + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + // see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019 + int64_t timezone = _timezone; + int32_t daylight = _daylight; + char **tzname = _tzname; +#endif +#endif + + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); + tz += daylight; + + /* + * format: + * (CST, +0800) + * (BST, +0100) + */ + + sprintf(outTimezone, "(%s, %s%02d00)", tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + *outDaylight = daylight; +} + +void taosGetSystemTimezone(char *outTimezone) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(outTimezone, "not configured"); + } else { + strcpy(outTimezone, tz); + } + +#elif defined(_TD_DARWIN_64) + char buf[4096] = {0}; + char *tz = NULL; + { + int n = readlink("/etc/localtime", buf, sizeof(buf)); + if (n < 0) { + printf("read /etc/localtime error, reason:%s", strerror(errno)); + return; + } + buf[n] = '\0'; + for (int i = n - 1; i >= 0; --i) { + if (buf[i] == '/') { + if (tz) { + tz = buf + i + 1; + break; + } + tz = buf + i + 1; + } + } + if (!tz || 0 == strchr(tz, '/')) { + printf("parsing /etc/localtime failed"); + return; + } + + setenv("TZ", tz, 1); + tzset(); + } + + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %+03ld00)", tz, tm1.tm_isdst ? tzname[daylight] : tzname[0], + -timezone / 3600); + +#else + /* + * NOTE: do not remove it. + * Enforce set the correct daylight saving time(DST) flag according + * to current time + */ + time_t tx1 = time(NULL); + struct tm tm1; + localtime_r(&tx1, &tm1); + + /* load time zone string from /etc/timezone */ + // FILE *f = fopen("/etc/timezone", "r"); + TdFilePtr pFile = taosOpenFile("/etc/timezone", TD_FILE_READ); + char buf[68] = {0}; + if (pFile != NULL) { + int len = taosReadFile(pFile, buf, 64); + if (len < 64 && taosGetErrorFile(pFile)) { + taosCloseFile(&pFile); + // printf("read /etc/timezone error, reason:%s", strerror(errno)); + return; + } + + taosCloseFile(&pFile); + + buf[sizeof(buf) - 1] = 0; + char *lineEnd = strstr(buf, "\n"); + if (lineEnd != NULL) { + *lineEnd = 0; + } + + // for CentOS system, /etc/timezone does not exist. Ignore the TZ environment variables + if (strlen(buf) > 0) { + setenv("TZ", buf, 1); + } + } + // get and set default timezone + tzset(); + + /* + * get CURRENT time zone. + * system current time zone is affected by daylight saving time(DST) + * + * e.g., the local time zone of London in DST is GMT+01:00, + * otherwise is GMT+00:00 + */ + int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + tz += daylight; + + /* + * format example: + * + * Asia/Shanghai (CST, +0800) + * Europe/London (BST, +0100) + */ + snprintf(outTimezone, TD_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz)); + +#endif +} diff --git a/source/util/inc/utilInt.h b/source/util/inc/utilInt.h deleted file mode 100644 index ae2bad792d8777283b9289516cb748ce21142651..0000000000000000000000000000000000000000 --- a/source/util/inc/utilInt.h +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_INT_H_ -#define _TD_UTIL_INT_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_INT_H_*/ \ No newline at end of file diff --git a/source/util/src/exception.c b/source/util/src/exception.c deleted file mode 100644 index 9740b9031bd876e05fb9e38ec28722732904c7df..0000000000000000000000000000000000000000 --- a/source/util/src/exception.c +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "exception.h" - -static threadlocal SExceptionNode* expList; - -void exceptionPushNode( SExceptionNode* node ) { - node->prev = expList; - expList = node; -} - -int32_t exceptionPopNode() { - SExceptionNode* node = expList; - expList = node->prev; - return node->code; -} - -void exceptionThrow( int32_t code ) { - expList->code = code; - longjmp( expList->jb, 1 ); -} - - - -static void cleanupWrapper_void_ptr_ptr( SCleanupAction* ca ) { - void (*func)( void*, void* ) = ca->func; - func( ca->arg1.Ptr, ca->arg2.Ptr ); -} - -static void cleanupWrapper_void_ptr_bool( SCleanupAction* ca ) { - void (*func)( void*, bool ) = ca->func; - func( ca->arg1.Ptr, ca->arg2.Bool ); -} - -static void cleanupWrapper_void_ptr( SCleanupAction* ca ) { - void (*func)( void* ) = ca->func; - func( ca->arg1.Ptr ); -} - -static void cleanupWrapper_int_int( SCleanupAction* ca ) { - int (*func)( int ) = ca->func; - func( ca->arg1.Int ); -} - -static void cleanupWrapper_void( SCleanupAction* ca ) { - void (*func)() = ca->func; - func(); -} - -static void cleanupWrapper_int_ptr( SCleanupAction* ca ) { - int (*func)( void* ) = ca->func; - func( ca->arg1.Ptr ); -} - -typedef void (*wrapper)(SCleanupAction*); -static wrapper wrappers[] = { - cleanupWrapper_void_ptr_ptr, - cleanupWrapper_void_ptr_bool, - cleanupWrapper_void_ptr, - cleanupWrapper_int_int, - cleanupWrapper_void, - cleanupWrapper_int_ptr, -}; - - -void cleanupPush_void_ptr_ptr( bool failOnly, void* func, void* arg1, void* arg2 ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 0; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg1; - ca->arg2.Ptr = arg2; -} - -void cleanupPush_void_ptr_bool( bool failOnly, void* func, void* arg1, bool arg2 ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 1; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg1; - ca->arg2.Bool = arg2; -} - -void cleanupPush_void_ptr( bool failOnly, void* func, void* arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 2; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg; -} - -void cleanupPush_int_int( bool failOnly, void* func, int arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 3; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Int = arg; -} - -void cleanupPush_void( bool failOnly, void* func ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 4; - ca->failOnly = failOnly; - ca->func = func; -} - -void cleanupPush_int_ptr( bool failOnly, void* func, void* arg ) { - assert( expList->numCleanupAction < expList->maxCleanupAction ); - - SCleanupAction *ca = expList->cleanupActions + expList->numCleanupAction++; - ca->wrapper = 5; - ca->failOnly = failOnly; - ca->func = func; - ca->arg1.Ptr = arg; -} - - -int32_t cleanupGetActionCount() { - return expList->numCleanupAction; -} - - -static void doExecuteCleanup( SExceptionNode* node, int32_t anchor, bool failed ) { - while( node->numCleanupAction > anchor ) { - --node->numCleanupAction; - SCleanupAction *ca = node->cleanupActions + node->numCleanupAction; - if( failed || !(ca->failOnly) ) { - wrappers[ca->wrapper]( ca ); - } - } -} - -void cleanupExecuteTo( int32_t anchor, bool failed ) { - doExecuteCleanup( expList, anchor, failed ); -} - -void cleanupExecute( SExceptionNode* node, bool failed ) { - doExecuteCleanup( node, 0, failed ); -} -bool cleanupExceedLimit() { - return expList->numCleanupAction >= expList->maxCleanupAction; -} diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index f8520b0742951cb6f63b6fc626326578282791e3..cb0a2d25a90a8999399e2e3bbfc3b4a4c838b0bd 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -13,30 +13,33 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "talgo.h" -#define doswap(__left, __right, __size, __buf) do {\ - memcpy((__buf), (__left), (__size));\ - memcpy((__left), (__right),(__size));\ - memcpy((__right), (__buf), (__size));\ -} while (0); +#define doswap(__left, __right, __size, __buf) \ + do { \ + memcpy((__buf), (__left), (__size)); \ + memcpy((__left), (__right), (__size)); \ + memcpy((__right), (__buf), (__size)); \ + } while (0); -static void median(void *src, size_t size, size_t s, size_t e, const void *param, __ext_compar_fn_t comparFn, void* buf) { +static void median(void *src, int64_t size, int64_t s, int64_t e, const void *param, __ext_compar_fn_t comparFn, + void *buf) { int32_t mid = ((int32_t)(e - s) >> 1u) + (int32_t)s; - + if (comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) == 1) { doswap(elePtrAt(src, size, mid), elePtrAt(src, size, s), size, buf); } - + if (comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, e), param) == 1) { doswap(elePtrAt(src, size, mid), elePtrAt(src, size, s), size, buf); doswap(elePtrAt(src, size, mid), elePtrAt(src, size, e), size, buf); } else if (comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) == 1) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } - - assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); + + assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 && + comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0); #ifdef _DEBUG_VIEW // tTagsPrints(src[s], pOrderDesc->pColumnModel, &pOrderDesc->orderIdx); @@ -45,8 +48,8 @@ static void median(void *src, size_t size, size_t s, size_t e, const void *param #endif } -static void tInsertSort(void *src, size_t size, int32_t s, int32_t e, const void *param, __ext_compar_fn_t comparFn, - void* buf) { +static void tInsertSort(void *src, int64_t size, int32_t s, int32_t e, const void *param, __ext_compar_fn_t comparFn, + void *buf) { for (int32_t i = s + 1; i <= e; ++i) { for (int32_t j = i; j > s; --j) { if (comparFn(elePtrAt(src, size, j), elePtrAt(src, size, j - 1), param) == -1) { @@ -58,117 +61,117 @@ static void tInsertSort(void *src, size_t size, int32_t s, int32_t e, const void } } -static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const void *param, __ext_compar_fn_t comparFn, - void* buf) { +static void tqsortImpl(void *src, int32_t start, int32_t end, int64_t size, const void *param, + __ext_compar_fn_t comparFn, void *buf) { // short array sort, incur another sort procedure instead of quick sort process const int32_t THRESHOLD_SIZE = 6; if (end - start + 1 <= THRESHOLD_SIZE) { tInsertSort(src, size, start, end, param, comparFn, buf); return; } - + median(src, size, start, end, param, comparFn, buf); - + int32_t s = start, e = end; int32_t endRightS = end, startLeftS = start; - + while (s < e) { while (e > s) { int32_t ret = comparFn(elePtrAt(src, size, e), elePtrAt(src, size, s), param); if (ret < 0) { break; } - - //move the data that equals to pivotal value to the right end of the list + + // move the data that equals to pivotal value to the right end of the list if (ret == 0 && e != endRightS) { doswap(elePtrAt(src, size, e), elePtrAt(src, size, endRightS), size, buf); endRightS--; } - + e--; } - + if (e != s) { doswap(elePtrAt(src, size, e), elePtrAt(src, size, s), size, buf); } - + while (s < e) { int32_t ret = comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param); if (ret > 0) { break; } - + if (ret == 0 && s != startLeftS) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, startLeftS), size, buf); startLeftS++; } s++; } - + if (e != s) { doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf); } } - + int32_t rightPartStart = e + 1; if (endRightS != end && e < end) { int32_t left = rightPartStart; int32_t right = end; - + while (right > endRightS && left <= endRightS) { doswap(elePtrAt(src, size, left), elePtrAt(src, size, right), size, buf); - + left++; right--; } - + rightPartStart += (end - endRightS); } - + int32_t leftPartEnd = e - 1; if (startLeftS != end && s > start) { int32_t left = start; int32_t right = leftPartEnd; - + while (left < startLeftS && right >= startLeftS) { doswap(elePtrAt(src, size, left), elePtrAt(src, size, right), size, buf); - + left++; right--; } - + leftPartEnd -= (startLeftS - start); } - + if (leftPartEnd > start) { tqsortImpl(src, start, leftPartEnd, size, param, comparFn, buf); } - + if (rightPartStart < end) { tqsortImpl(src, rightPartStart, end, size, param, comparFn, buf); } } -void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) { - char *buf = calloc(1, size); // prepare the swap buffer +void taosqsort(void *src, int64_t numOfElem, int64_t size, const void *param, __ext_compar_fn_t comparFn) { + char *buf = calloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); tfree(buf); } -void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) { +void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size, __compar_fn_t compar, int32_t flags) { // TODO: need to check the correctness of this function - int l = 0; - int r = (int)nmemb; - int idx = 0; - int comparison; - + int32_t l = 0; + int32_t r = (int32_t)nmemb; + int32_t idx = 0; + int32_t comparison; + if (flags == TD_EQ) { return bsearch(key, base, nmemb, size, compar); } else if (flags == TD_GE) { if (nmemb <= 0) return NULL; if ((*compar)(key, elePtrAt(base, size, 0)) <= 0) return elePtrAt(base, size, 0); if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) > 0) return NULL; - + while (l < r) { idx = (l + r) / 2; comparison = (*compar)(key, elePtrAt(base, size, idx)); @@ -180,7 +183,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx); } } - + if ((*compar)(key, elePtrAt(base, size, idx)) < 0) { return elePtrAt(base, size, idx); } else { @@ -194,7 +197,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, if (nmemb <= 0) return NULL; if ((*compar)(key, elePtrAt(base, size, nmemb - 1)) >= 0) return elePtrAt(base, size, nmemb - 1); if ((*compar)(key, elePtrAt(base, size, 0)) < 0) return NULL; - + while (l < r) { idx = (l + r) / 2; comparison = (*compar)(key, elePtrAt(base, size, idx)); @@ -206,7 +209,7 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx); } } - + if ((*compar)(key, elePtrAt(base, size, idx)) > 0) { return elePtrAt(base, size, idx); } else { @@ -216,20 +219,20 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return elePtrAt(base, size, idx - 1); } } - + } else { assert(0); return NULL; } - + return NULL; } -void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) -{ - int32_t parent; - int32_t child; - char *buf; +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, + __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + int32_t parent; + int32_t child; + char *buf; if (base && size > 0 && compar) { parent = start; @@ -244,7 +247,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const if (maxroot) { while (child <= end) { - if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { + if (child + 1 <= end && + (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { child++; } @@ -263,7 +267,8 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } } else { while (child <= end) { - if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { + if (child + 1 <= end && + (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { child++; } @@ -288,9 +293,9 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const } } -void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) -{ - int32_t i; +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, + const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + int32_t i; if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { @@ -298,15 +303,14 @@ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, } } -/* - char *buf = calloc(1, size); + /* + char *buf = calloc(1, size); - for (i = len - 1; i > 0; i--) { - doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); - taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); - } + for (i = len - 1; i > 0; i--) { + doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); + taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); + } - tfree(buf); -*/ + tfree(buf); + */ } - diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 3e6f97c0c3bcdc30d39f08dd469babd3199a1fa9..1127a212559ab832a6afdb367c8ab5d4b46a6877 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -13,9 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tarray.h" -#include "talgo.h" SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); @@ -54,7 +53,7 @@ static int32_t taosArrayResize(SArray* pArray) { pArray->pData = tmp; pArray->capacity = size; - + return 0; } @@ -75,12 +74,12 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t newCap) { return 0; } -void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { +void* taosArrayAddBatch(SArray* pArray, const void* pData, int32_t nEles) { if (pArray == NULL || pData == NULL) { return NULL; } - if(taosArrayEnsureCap(pArray, pArray->size + nEles) != 0){ + if (taosArrayEnsureCap(pArray, pArray->size + nEles) != 0) { return NULL; } @@ -91,7 +90,7 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { return dst; } -void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems) { +void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems) { assert(pArray != NULL && pData != NULL); if (numOfElems <= 0) { return; @@ -104,7 +103,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle } int32_t i = pData[0] + 1, j = 0; - while(i < size) { + while (i < size) { if (j == numOfElems - 1) { break; } @@ -133,7 +132,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle pArray->size -= numOfElems; } -void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)) { +void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)) { assert(pArray); size_t size = pArray->size; @@ -142,7 +141,7 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } int32_t pos = 0; - for(int32_t i = 1; i < size; ++i) { + for (int32_t i = 1; i < size; ++i) { char* p1 = taosArrayGet(pArray, pos); char* p2 = taosArrayGet(pArray, i); @@ -164,7 +163,7 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } if (fp != NULL) { - for(int32_t i = pos + 1; i < pArray->size; ++i) { + for (int32_t i = pos + 1; i < pArray->size; ++i) { void* p = taosArrayGet(pArray, i); fp(p); } @@ -174,11 +173,11 @@ void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp) } void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { - return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); + return taosArrayAddBatch(pArray, pInput->pData, (int32_t)taosArrayGetSize(pInput)); } void* taosArrayPop(SArray* pArray) { - assert( pArray != NULL ); + assert(pArray != NULL); if (pArray->size == 0) { return NULL; @@ -194,15 +193,13 @@ void* taosArrayGet(const SArray* pArray, size_t index) { void* taosArrayGetP(const SArray* pArray, size_t index) { assert(index < pArray->size); - + void* d = TARRAY_GET_ELEM(pArray, index); - + return *(void**)d; } -void* taosArrayGetLast(const SArray* pArray) { - return TARRAY_GET_ELEM(pArray, pArray->size - 1); -} +void* taosArrayGetLast(const SArray* pArray) { return TARRAY_GET_ELEM(pArray, pArray->size - 1); } size_t taosArrayGetSize(const SArray* pArray) { if (pArray == NULL) { @@ -227,7 +224,7 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { if (pArray->size >= pArray->capacity) { int32_t ret = taosArrayResize(pArray); - + if (ret < 0) { return NULL; } @@ -240,7 +237,7 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { memcpy(dst, pData, pArray->elemSize); pArray->size += 1; - + return dst; } @@ -252,7 +249,7 @@ void taosArraySet(SArray* pArray, size_t index, void* pData) { void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { assert(cnt <= pArray->size); pArray->size = pArray->size - cnt; - if(pArray->size == 0) { + if (pArray->size == 0) { return; } memmove(pArray->pData, (char*)pArray->pData + cnt * pArray->elemSize, pArray->size * pArray->elemSize); @@ -265,14 +262,15 @@ void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { void taosArrayRemove(SArray* pArray, size_t index) { assert(index < pArray->size); - + if (index == pArray->size - 1) { taosArrayPop(pArray); return; } - + size_t remain = pArray->size - index - 1; - memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + 1) * pArray->elemSize, remain * pArray->elemSize); + memmove((char*)pArray->pData + index * pArray->elemSize, (char*)pArray->pData + (index + 1) * pArray->elemSize, + remain * pArray->elemSize); pArray->size -= 1; } @@ -288,13 +286,13 @@ SArray* taosArrayFromList(const void* src, size_t size, size_t elemSize) { SArray* taosArrayDup(const SArray* pSrc) { assert(pSrc != NULL); - - if (pSrc->size == 0) { // empty array list + + if (pSrc->size == 0) { // empty array list return taosArrayInit(8, pSrc->elemSize); } - + SArray* dst = taosArrayInit(pSrc->size, pSrc->elemSize); - + memcpy(dst->pData, pSrc->pData, pSrc->elemSize * pSrc->size); dst->size = pSrc->size; return dst; @@ -324,7 +322,7 @@ void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) { return; } - for(int32_t i = 0; i < pArray->size; ++i) { + for (int32_t i = 0; i < pArray->size; ++i) { fp(TARRAY_GET_ELEM(pArray, i)); } @@ -338,14 +336,14 @@ void taosArraySort(SArray* pArray, __compar_fn_t compar) { qsort(pArray->pData, pArray->size, pArray->elemSize, compar); } -void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { +void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) { assert(pArray != NULL && comparFn != NULL); assert(key != NULL); return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags); } -int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) { +int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t comparFn, int32_t flags) { void* item = taosArraySearch(pArray, key, comparFn, flags); return item == NULL ? -1 : (int32_t)((char*)item - (char*)pArray->pData) / pArray->elemSize; } @@ -355,7 +353,7 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) { qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn); } -char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) { +char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int32_t flags) { assert(pArray != NULL); assert(key != NULL); @@ -366,47 +364,53 @@ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t return *(char**)p; } -static int taosArrayPartition(SArray *pArray, int i, int j, __ext_compar_fn_t fn, const void *userData) { - void* key = taosArrayGetP(pArray, i); +static int32_t taosArrayPartition(SArray* pArray, int32_t i, int32_t j, __ext_compar_fn_t fn, const void* userData) { + void* key = taosArrayGetP(pArray, i); while (i < j) { - while (i < j && fn(taosArrayGetP(pArray, j), key, userData) >= 0) { j--; } - if (i < j) { - void *a = taosArrayGetP(pArray, j); + while (i < j && fn(taosArrayGetP(pArray, j), key, userData) >= 0) { + j--; + } + if (i < j) { + void* a = taosArrayGetP(pArray, j); taosArraySet(pArray, i, &a); } - while (i < j && fn(taosArrayGetP(pArray, i), key, userData) <= 0) { i++;} + while (i < j && fn(taosArrayGetP(pArray, i), key, userData) <= 0) { + i++; + } if (i < j) { - void *a = taosArrayGetP(pArray, i); + void* a = taosArrayGetP(pArray, i); taosArraySet(pArray, j, &a); } } - taosArraySet(pArray, i, &key); + taosArraySet(pArray, i, &key); return i; } -static void taosArrayQuicksortHelper(SArray *pArray, int low, int high, __ext_compar_fn_t fn, const void *param) { +static void taosArrayQuicksortHelper(SArray* pArray, int32_t low, int32_t high, __ext_compar_fn_t fn, + const void* param) { if (low < high) { - int idx = taosArrayPartition(pArray, low, high, fn, param); + int32_t idx = taosArrayPartition(pArray, low, high, fn, param); taosArrayQuicksortHelper(pArray, low, idx - 1, fn, param); taosArrayQuicksortHelper(pArray, idx + 1, high, fn, param); - } + } } -static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { +static void taosArrayQuickSort(SArray* pArray, __ext_compar_fn_t fn, const void* param) { if (pArray->size <= 1) { return; - } - taosArrayQuicksortHelper(pArray, 0, (int)(taosArrayGetSize(pArray) - 1), fn, param); + } + taosArrayQuicksortHelper(pArray, 0, (int32_t)(taosArrayGetSize(pArray) - 1), fn, param); } -static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void *param) { + +static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void* param) { if (pArray->size <= 1) { return; - } - for (int i = 1; i <= pArray->size - 1; ++i) { - for (int j = i; j > 0; --j) { + } + for (int32_t i = 1; i <= pArray->size - 1; ++i) { + for (int32_t j = i; j > 0; --j) { if (fn(taosArrayGetP(pArray, j), taosArrayGetP(pArray, j - 1), param) == -1) { - void *a = taosArrayGetP(pArray, j); - void *b = taosArrayGetP(pArray, j - 1); + void* a = taosArrayGetP(pArray, j); + void* b = taosArrayGetP(pArray, j - 1); taosArraySet(pArray, j - 1, &a); taosArraySet(pArray, j, &b); } else { @@ -415,11 +419,10 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } } return; - -} -// order array -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param) { - taosArrayGetSize(pArray) > 8 ? - taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); -} -//TODO(yihaoDeng) add order array +} + +// order array +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { + taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); +} +// TODO(yihaoDeng) add order array diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index 1b1f53df17d83313744d0ea528bfb013878eb53d..c5119e8b2d43cce3e0cc0781c54206da8f9f0bea 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -13,15 +13,15 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE +#include "tbase64.h" -// deprecated this file for bug prone -// base64 encode static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -char *base64_encode(const unsigned char *value, int vlen) { - unsigned char oval = 0; - char * result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); - char * out = result; + +char *base64_encode(const uint8_t *value, int32_t vlen) { + uint8_t oval = 0; + char *result = (char *)malloc((size_t)(vlen * 4) / 3 + 10); + char *out = result; while (vlen >= 3) { *out++ = basis_64[value[0] >> 2]; *out++ = basis_64[((value[0] << 4) & 0x30) | (value[1] >> 4)]; @@ -42,8 +42,8 @@ char *base64_encode(const unsigned char *value, int vlen) { return result; } -// base64 decode #define CHAR64(c) (((c) < 0 || (c) > 127) ? -1 : index_64[(c)]) + static signed char index_64[128] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54, 55, @@ -51,10 +51,10 @@ static signed char index_64[128] = { 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1}; -unsigned char *base64_decode(const char *value, int inlen, int *outlen) { - int c1, c2, c3, c4; - unsigned char *result = (unsigned char *)malloc((size_t)(inlen * 3) / 4 + 1); - unsigned char *out = result; +uint8_t *base64_decode(const char *value, int32_t inlen, int32_t *outlen) { + int32_t c1, c2, c3, c4; + uint8_t *result = (uint8_t *)malloc((size_t)(inlen * 3) / 4 + 1); + uint8_t *out = result; *outlen = 0; @@ -80,13 +80,13 @@ unsigned char *base64_decode(const char *value, int inlen, int *outlen) { if ((c4 != '=') && (CHAR64(c4) == -1)) goto base64_decode_error; value += 4; - *out++ = (unsigned char)((CHAR64(c1) << 2) | (CHAR64(c2) >> 4)); + *out++ = (uint8_t)((CHAR64(c1) << 2) | (CHAR64(c2) >> 4)); *outlen += 1; if (c3 != '=') { - *out++ = (unsigned char)(((CHAR64(c2) << 4) & 0xf0) | (CHAR64(c3) >> 2)); + *out++ = (uint8_t)(((CHAR64(c2) << 4) & 0xf0) | (CHAR64(c3) >> 2)); *outlen += 1; if (c4 != '=') { - *out++ = (unsigned char)(((CHAR64(c3) << 6) & 0xc0) | CHAR64(c4)); + *out++ = (uint8_t)(((CHAR64(c3) << 6) & 0xc0) | CHAR64(c4)); *outlen += 1; } } diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index 0456d6a2eecca33a9509767dda27406bdf887e54..dc8c4b70c5107358a742e16ceef67584a111ff69 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -14,11 +14,8 @@ */ #define _DEFAULT_SOURCE - #include "tbuffer.h" -#include "exception.h" -#include "os.h" -//#include "taoserror.h" +#include "texception.h" typedef union Un4B { uint32_t ui; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 621f103207e0161a1189a63d0fc9f108f322afca..64d822f750837d2b60b64c927b46288b149588ba 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -14,11 +14,10 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" +#include "tcache.h" +#include "tlog.h" #include "ttimer.h" #include "tutil.h" -#include "tcache.h" static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #if defined(LINUX) @@ -62,15 +61,15 @@ static void doCleanupDataCache(SCacheObj *pCacheObj); * refresh cache to remove data in both hash list and trash, if any nodes' refcount == 0, every pCacheObj->refreshTime * @param handle Cache object handle */ -static void* taosCacheTimedRefresh(void *handle); +static void *taosCacheTimedRefresh(void *handle); -static pthread_t cacheRefreshWorker = {0}; -static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; -static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; -static SArray* pCacheArrayList = NULL; -static bool stopRefreshWorker = false; -static bool refreshWorkerNormalStopped = false; -static bool refreshWorkerUnexpectedStopped = false; +static pthread_t cacheRefreshWorker = {0}; +static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT; +static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER; +static SArray *pCacheArrayList = NULL; +static bool stopRefreshWorker = false; +static bool refreshWorkerNormalStopped = false; +static bool refreshWorkerUnexpectedStopped = false; static void doInitRefreshThread(void) { pCacheArrayList = taosArrayInit(4, POINTER_BYTES); @@ -83,7 +82,7 @@ static void doInitRefreshThread(void) { pthread_attr_destroy(&thattr); } -pthread_t doRegisterCacheObj(SCacheObj* pCacheObj) { +pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) { pthread_once(&cacheThreadInit, doInitRefreshThread); pthread_mutex_lock(&guard); @@ -102,7 +101,8 @@ pthread_t doRegisterCacheObj(SCacheObj* pCacheObj) { * @param lifespan total survial expiredTime from now * @return SCacheDataNode */ -static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration); +static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, + uint64_t duration); /** * addedTime object node into trash, and this object is closed for referencing if it is addedTime to trash @@ -146,18 +146,18 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo free(pNode); } -static FORCE_INLINE STrashElem* doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) { - if (pElem->pData->signature != (uint64_t) pElem->pData) { +static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STrashElem *pElem) { + if (pElem->pData->signature != (uint64_t)pElem->pData) { uWarn("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); return NULL; } - STrashElem* next = pElem->next; + STrashElem *next = pElem->next; pCacheObj->numOfElemsInTrash--; if (pElem->prev) { pElem->prev->next = pElem->next; - } else { // pnode is the header, update header + } else { // pnode is the header, update header pCacheObj->pTrash = pElem->next; } @@ -172,7 +172,7 @@ static FORCE_INLINE STrashElem* doRemoveElemInTrashcan(SCacheObj* pCacheObj, STr return next; } -static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem *pElem) { +static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem *pElem) { if (pCacheObj->freeFp) { pCacheObj->freeFp(pElem->pData->data); } @@ -181,19 +181,20 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem free(pElem); } -SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, const char* cacheName) { - const int32_t SLEEP_DURATION = 500; //500 ms +SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, + const char *cacheName) { + const int32_t SLEEP_DURATION = 500; // 500 ms if (refreshTimeInSeconds <= 0) { return NULL; } - + SCacheObj *pCacheObj = (SCacheObj *)calloc(1, sizeof(SCacheObj)); if (pCacheObj == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - + pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { @@ -201,17 +202,17 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - + // set free cache node callback function - pCacheObj->freeFp = fn; + pCacheObj->freeFp = fn; pCacheObj->refreshTime = refreshTimeInSeconds * 1000; - pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; + pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION; pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access if (__cache_lock_init(pCacheObj) != 0) { taosHashCleanup(pCacheObj->pHashTable); free(pCacheObj); - + uError("failed to init lock, reason:%s", strerror(errno)); return NULL; } @@ -220,7 +221,8 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, + int32_t durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } @@ -242,8 +244,8 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); } else { // duplicated key exists while (1) { - SCacheDataNode* p = NULL; -// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); + SCacheDataNode *p = NULL; + // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen); // add to trashcan @@ -283,10 +285,10 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return pNode1->data; } -static void incRefFn(void* ptNode) { +static void incRefFn(void *ptNode) { assert(ptNode != NULL); - SCacheDataNode** p = (SCacheDataNode**) ptNode; + SCacheDataNode **p = (SCacheDataNode **)ptNode; assert(T_REF_VAL_GET(*p) >= 0); int32_t ret = T_REF_INC(*p); @@ -303,15 +305,16 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return NULL; } - SCacheDataNode* ptNode = NULL; + SCacheDataNode *ptNode = NULL; taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode); -// taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); + // taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode); - void* pData = (ptNode != NULL)? ptNode->data:NULL; + void *pData = (ptNode != NULL) ? ptNode->data : NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(ptNode)); + uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, + T_REF_VAL_GET(ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -323,10 +326,10 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; - + size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset); - + if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; @@ -342,22 +345,22 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL; - + size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset); - + if (ptNode->signature != (uint64_t)ptNode) { uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode); return NULL; } - + assert(T_REF_VAL_GET(ptNode) >= 1); - + char *d = *data; - + // clear its reference to old area *data = NULL; - + return d; } @@ -371,13 +374,12 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { return; } - // The operation of removal from hash table and addition to trashcan is not an atomic operation, // therefore the check for the empty of both the hash table and the trashcan has a race condition. // It happens when there is only one object in the cache, and two threads which has referenced this object // start to free the it simultaneously [TD-1569]. size_t offset = offsetof(SCacheDataNode, data); - + SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); if (pNode->signature != (uint64_t)pNode) { uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode); @@ -391,20 +393,20 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (pCacheObj->extendLifespan && (!inTrashcan) && (!_remove)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); - uDebug("cache:%s, data:%p extend expire time: %"PRId64, pCacheObj->name, pNode->data, pNode->expireTime); + uDebug("cache:%s, data:%p extend expire time: %" PRId64, pCacheObj->name, pNode->data, pNode->expireTime); } if (_remove) { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - char* key = pNode->key; - char* d = pNode->data; + char *key = pNode->key; + char *d = pNode->data; int32_t ref = T_REF_VAL_GET(pNode); uDebug("cache:%s, key:%p, %p is released, refcnt:%d, in trashcan:%d", pCacheObj->name, key, d, ref - 1, inTrashcan); /* - * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users - * releasing this resources. + * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all + * users releasing this resources. * * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread * that tries to do the same thing. @@ -433,16 +435,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread // when reaches here. SCacheDataNode *p = NULL; - int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); -// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void *)); + int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + // int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void + // *)); ref = T_REF_DEC(pNode); // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. // note that the remove operation can be executed only once. if (ret == 0) { if (p != pNode) { - uDebug( "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " - "others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); + uDebug( + "cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by " + "others already", + pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data); assert(p->pTNodeHeader == NULL); taosAddToTrashcan(pCacheObj, p); @@ -468,35 +473,36 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } } } else { - uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", - pCacheObj->name, pNode->key, pNode->data, ref); + uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name, + pNode->key, pNode->data, ref); } } } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - char* key = pNode->key; - char* p = pNode->data; - -// int32_t ref = T_REF_VAL_GET(pNode); -// -// if (ref == 1 && inTrashcan) { -// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be -// // destroyed by refresh worker if decrease ref count before removing it from linked-list. -// assert(pNode->pTNodeHeader->pData == pNode); -// -// __cache_wr_lock(pCacheObj); -// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); -// __cache_unlock(pCacheObj); -// -// ref = T_REF_DEC(pNode); -// assert(ref == 0); -// -// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); -// } else { -// ref = T_REF_DEC(pNode); -// assert(ref >= 0); -// } + char *key = pNode->key; + char *p = pNode->data; + + // int32_t ref = T_REF_VAL_GET(pNode); + // + // if (ref == 1 && inTrashcan) { + // // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may + // be + // // destroyed by refresh worker if decrease ref count before removing it from linked-list. + // assert(pNode->pTNodeHeader->pData == pNode); + // + // __cache_wr_lock(pCacheObj); + // doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); + // __cache_unlock(pCacheObj); + // + // ref = T_REF_DEC(pNode); + // assert(ref == 0); + // + // doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); + // } else { + // ref = T_REF_DEC(pNode); + // assert(ref >= 0); + // } int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan); @@ -504,21 +510,21 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } typedef struct SHashTravSupp { - SCacheObj* pCacheObj; - int64_t time; + SCacheObj *pCacheObj; + int64_t time; __cache_trav_fn_t fp; - void* param1; + void *param1; } SHashTravSupp; -static bool travHashTableEmptyFn(void* param, void* data) { - SHashTravSupp* ps = (SHashTravSupp*) param; - SCacheObj* pCacheObj= ps->pCacheObj; +static bool travHashTableEmptyFn(void *param, void *data) { + SHashTravSupp *ps = (SHashTravSupp *)param; + SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode *pNode = *(SCacheDataNode **) data; + SCacheDataNode *pNode = *(SCacheDataNode **)data; if (T_REF_VAL_GET(pNode) == 0) { taosCacheReleaseNode(pCacheObj, pNode); - } else { // do add to trashcan + } else { // do add to trashcan taosAddToTrashcan(pCacheObj, pNode); } @@ -529,7 +535,7 @@ static bool travHashTableEmptyFn(void* param, void* data) { void taosCacheEmpty(SCacheObj *pCacheObj) { SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); taosTrashcanEmpty(pCacheObj, false); } @@ -542,9 +548,9 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { // wait for the refresh thread quit before destroying the cache object. // But in the dll, the child thread will be killed before atexit takes effect. - while(atomic_load_8(&pCacheObj->deleting) != 0) { - if (refreshWorkerNormalStopped) break; - if (refreshWorkerUnexpectedStopped) return; + while (atomic_load_8(&pCacheObj->deleting) != 0) { + if (refreshWorkerNormalStopped) break; + if (refreshWorkerUnexpectedStopped) return; taosMsleep(50); } @@ -568,11 +574,11 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char * memcpy(pNewNode->key, key, keyLen); - pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); - pNewNode->lifespan = duration; - pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; - pNewNode->signature = (uint64_t)pNewNode; - pNewNode->size = (uint32_t)totalSize; + pNewNode->addedTime = (uint64_t)taosGetTimestampMs(); + pNewNode->lifespan = duration; + pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan; + pNewNode->signature = (uint64_t)pNewNode; + pNewNode->size = (uint32_t)totalSize; return pNewNode; } @@ -610,16 +616,17 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { if (pCacheObj->numOfElemsInTrash == 0) { if (pCacheObj->pTrash != NULL) { pCacheObj->pTrash = NULL; - uError("cache:%s, key:inconsistency data in cache, numOfElem in trashcan:%d", pCacheObj->name, pCacheObj->numOfElemsInTrash); + uError("cache:%s, key:inconsistency data in cache, numOfElem in trashcan:%d", pCacheObj->name, + pCacheObj->numOfElemsInTrash); } __cache_unlock(pCacheObj); return; } - const char* stat[] = {"false", "true"}; + const char *stat[] = {"false", "true"}; uDebug("cache:%s start to cleanup trashcan, numOfElem in trashcan:%d, free:%s", pCacheObj->name, - pCacheObj->numOfElemsInTrash, (force? stat[1]:stat[0])); + pCacheObj->numOfElemsInTrash, (force ? stat[1] : stat[0])); STrashElem *pElem = pCacheObj->pTrash; while (pElem) { @@ -627,8 +634,8 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { assert(pElem->next != pElem && pElem->prev != pElem); if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { - uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, pElem->pData->data, - pCacheObj->numOfElemsInTrash - 1); + uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, + pElem->pData->data, pCacheObj->numOfElemsInTrash - 1); doRemoveElemInTrashcan(pCacheObj, pElem); doDestroyTrashcanElem(pCacheObj, pElem); @@ -642,25 +649,25 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { } void doCleanupDataCache(SCacheObj *pCacheObj) { -// SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); + // SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); taosTrashcanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); - + tfree(pCacheObj->name); memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } -bool travHashTableFn(void* param, void* data) { - SHashTravSupp* ps = (SHashTravSupp*) param; - SCacheObj* pCacheObj= ps->pCacheObj; +bool travHashTableFn(void *param, void *data) { + SHashTravSupp *ps = (SHashTravSupp *)param; + SCacheObj *pCacheObj = ps->pCacheObj; - SCacheDataNode* pNode = *(SCacheDataNode **) data; + SCacheDataNode *pNode = *(SCacheDataNode **)data; if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { taosCacheReleaseNode(pCacheObj, pNode); @@ -676,30 +683,30 @@ bool travHashTableFn(void* param, void* data) { return true; } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_trav_fn_t fp, void* param1) { +static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { assert(pCacheObj != NULL); SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; -// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); + // taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } void taosCacheRefreshWorkerUnexpectedStopped(void) { - if(!refreshWorkerNormalStopped) { - refreshWorkerUnexpectedStopped=true; + if (!refreshWorkerNormalStopped) { + refreshWorkerUnexpectedStopped = true; } } -void* taosCacheTimedRefresh(void *handle) { +void *taosCacheTimedRefresh(void *handle) { assert(pCacheArrayList != NULL); uDebug("cache refresh thread starts"); setThreadName("cacheRefresh"); - const int32_t SLEEP_DURATION = 500; //500 ms - int64_t count = 0; + const int32_t SLEEP_DURATION = 500; // 500 ms + int64_t count = 0; atexit(taosCacheRefreshWorkerUnexpectedStopped); - while(1) { + while (1) { taosMsleep(SLEEP_DURATION); if (stopRefreshWorker) { goto _end; @@ -711,9 +718,9 @@ void* taosCacheTimedRefresh(void *handle) { count += 1; - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { pthread_mutex_lock(&guard); - SCacheObj* pCacheObj = taosArrayGetP(pCacheArrayList, i); + SCacheObj *pCacheObj = taosArrayGetP(pCacheArrayList, i); if (pCacheObj == NULL) { uError("object is destroyed. ignore and try next"); @@ -726,8 +733,8 @@ void* taosCacheTimedRefresh(void *handle) { taosArrayRemove(pCacheArrayList, i); size = taosArrayGetSize(pCacheArrayList); - uDebug("%s is destroying, remove it from refresh list, remain cache obj:%"PRIzu, pCacheObj->name, size); - pCacheObj->deleting = 0; //reset the deleting flag to enable pCacheObj to continue releasing resources. + uDebug("%s is destroying, remove it from refresh list, remain cache obj:%" PRIzu, pCacheObj->name, size); + pCacheObj->deleting = 0; // reset the deleting flag to enable pCacheObj to continue releasing resources. pthread_mutex_unlock(&guard); continue; @@ -757,18 +764,18 @@ void* taosCacheTimedRefresh(void *handle) { } } - _end: +_end: taosArrayDestroy(pCacheArrayList); pCacheArrayList = NULL; pthread_mutex_destroy(&guard); - refreshWorkerNormalStopped=true; + refreshWorkerNormalStopped = true; uDebug("cache refresh thread quits"); return NULL; } -void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) { +void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1) { if (pCacheObj == NULL) { return; } @@ -777,6 +784,4 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void* param1) doCacheRefresh(pCacheObj, now, fp, param1); } -void taosStopCacheRefreshWorker(void) { - stopRefreshWorker = true; -} \ No newline at end of file +void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; } \ No newline at end of file diff --git a/source/util/src/compare.c b/source/util/src/tcompare.c similarity index 65% rename from source/util/src/compare.c rename to source/util/src/tcompare.c index 1b1fa417546c611eb3b97e536b5cd8cfb569e8d1..062d144933136891acc98b4ba07b43d09c8a9f69 100644 --- a/source/util/src/compare.c +++ b/source/util/src/tcompare.c @@ -16,15 +16,12 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE -#include "os.h" - -#include "compare.h" +#include "tcompare.h" #include "regex.h" +#include "tdef.h" #include "thash.h" +#include "tlog.h" #include "types.h" -#include "ulog.h" -#include "tdef.h" -#include "taos.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -58,16 +55,14 @@ int32_t setChkNotInBytes8(const void *pLeft, const void *pRight) { return NULL == taosHashGet((SHashObj *)pRight, pLeft, 8) ? 1 : 0; } - -int32_t compareChkInString(const void *pLeft, const void* pRight) { +int32_t compareChkInString(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } -int32_t compareChkNotInString(const void *pLeft, const void* pRight) { +int32_t compareChkNotInString(const void *pLeft, const void *pRight) { return NULL == taosHashGet((SHashObj *)pRight, varDataVal(pLeft), varDataLen(pLeft)) ? 1 : 0; } - int32_t compareInt8Val(const void *pLeft, const void *pRight) { int8_t left = GET_INT8_VAL(pLeft), right = GET_INT8_VAL(pRight); if (left > right) return 1; @@ -75,9 +70,7 @@ int32_t compareInt8Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { - return compareInt8Val(pRight, pLeft); -} +int32_t compareInt8ValDesc(const void *pLeft, const void *pRight) { return compareInt8Val(pRight, pLeft); } int32_t compareInt16Val(const void *pLeft, const void *pRight) { int16_t left = GET_INT16_VAL(pLeft), right = GET_INT16_VAL(pRight); @@ -86,9 +79,7 @@ int32_t compareInt16Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt16ValDesc(const void* pLeft, const void* pRight) { - return compareInt16Val(pRight, pLeft); -} +int32_t compareInt16ValDesc(const void *pLeft, const void *pRight) { return compareInt16Val(pRight, pLeft); } int32_t compareInt32Val(const void *pLeft, const void *pRight) { int32_t left = GET_INT32_VAL(pLeft), right = GET_INT32_VAL(pRight); @@ -97,9 +88,7 @@ int32_t compareInt32Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt32ValDesc(const void* pLeft, const void* pRight) { - return compareInt32Val(pRight, pLeft); -} +int32_t compareInt32ValDesc(const void *pLeft, const void *pRight) { return compareInt32Val(pRight, pLeft); } int32_t compareInt64Val(const void *pLeft, const void *pRight) { int64_t left = GET_INT64_VAL(pLeft), right = GET_INT64_VAL(pRight); @@ -108,9 +97,7 @@ int32_t compareInt64Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareInt64ValDesc(const void* pLeft, const void* pRight) { - return compareInt64Val(pRight, pLeft); -} +int32_t compareInt64ValDesc(const void *pLeft, const void *pRight) { return compareInt64Val(pRight, pLeft); } int32_t compareUint32Val(const void *pLeft, const void *pRight) { uint32_t left = GET_UINT32_VAL(pLeft), right = GET_UINT32_VAL(pRight); @@ -119,9 +106,7 @@ int32_t compareUint32Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint32ValDesc(const void* pLeft, const void* pRight) { - return compareUint32Val(pRight, pLeft); -} +int32_t compareUint32ValDesc(const void *pLeft, const void *pRight) { return compareUint32Val(pRight, pLeft); } int32_t compareUint64Val(const void *pLeft, const void *pRight) { uint64_t left = GET_UINT64_VAL(pLeft), right = GET_UINT64_VAL(pRight); @@ -130,9 +115,7 @@ int32_t compareUint64Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint64ValDesc(const void* pLeft, const void* pRight) { - return compareUint64Val(pRight, pLeft); -} +int32_t compareUint64ValDesc(const void *pLeft, const void *pRight) { return compareUint64Val(pRight, pLeft); } int32_t compareUint16Val(const void *pLeft, const void *pRight) { uint16_t left = GET_UINT16_VAL(pLeft), right = GET_UINT16_VAL(pRight); @@ -141,20 +124,16 @@ int32_t compareUint16Val(const void *pLeft, const void *pRight) { return 0; } -int32_t compareUint16ValDesc(const void* pLeft, const void* pRight) { - return compareUint16Val(pRight, pLeft); -} +int32_t compareUint16ValDesc(const void *pLeft, const void *pRight) { return compareUint16Val(pRight, pLeft); } -int32_t compareUint8Val(const void* pLeft, const void* pRight) { +int32_t compareUint8Val(const void *pLeft, const void *pRight) { uint8_t left = GET_UINT8_VAL(pLeft), right = GET_UINT8_VAL(pRight); if (left > right) return 1; if (left < right) return -1; return 0; } -int32_t compareUint8ValDesc(const void* pLeft, const void* pRight) { - return compareUint8Val(pRight, pLeft); -} +int32_t compareUint8ValDesc(const void *pLeft, const void *pRight) { return compareUint8Val(pRight, pLeft); } int32_t compareFloatVal(const void *pLeft, const void *pRight) { float p1 = GET_FLOAT_VAL(pLeft); @@ -174,12 +153,10 @@ int32_t compareFloatVal(const void *pLeft, const void *pRight) { if (FLT_EQUAL(p1, p2)) { return 0; } - return FLT_GREATER(p1, p2) ? 1: -1; + return FLT_GREATER(p1, p2) ? 1 : -1; } -int32_t compareFloatValDesc(const void* pLeft, const void* pRight) { - return compareFloatVal(pRight, pLeft); -} +int32_t compareFloatValDesc(const void *pLeft, const void *pRight) { return compareFloatVal(pRight, pLeft); } int32_t compareDoubleVal(const void *pLeft, const void *pRight) { double p1 = GET_DOUBLE_VAL(pLeft); @@ -199,30 +176,28 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight) { if (FLT_EQUAL(p1, p2)) { return 0; } - return FLT_GREATER(p1, p2) ? 1: -1; + return FLT_GREATER(p1, p2) ? 1 : -1; } -int32_t compareDoubleValDesc(const void* pLeft, const void* pRight) { - return compareDoubleVal(pRight, pLeft); -} +int32_t compareDoubleValDesc(const void *pLeft, const void *pRight) { return compareDoubleVal(pRight, pLeft); } int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); if (len1 != len2) { - return len1 > len2? 1:-1; + return len1 > len2 ? 1 : -1; } else { int32_t ret = strncmp(varDataVal(pLeft), varDataVal(pRight), len1); if (ret == 0) { return 0; } else { - return ret > 0 ? 1:-1; + return ret > 0 ? 1 : -1; } } } -int32_t compareLenPrefixedStrDesc(const void* pLeft, const void* pRight) { +int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedStr(pRight, pLeft); } @@ -231,9 +206,9 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len2 = varDataLen(pRight); if (len1 != len2) { - return len1 > len2? 1:-1; + return len1 > len2 ? 1 : -1; } else { - int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1); + int32_t ret = memcmp((wchar_t *)pLeft, (wchar_t *)pRight, len1); if (ret == 0) { return 0; } else { @@ -242,7 +217,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { } } -int32_t compareLenPrefixedWStrDesc(const void* pLeft, const void* pRight) { +int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight) { return compareLenPrefixedWStr(pRight, pLeft); } @@ -302,10 +277,13 @@ int32_t patternMatch(const char *patterStr, const char *str, size_t size, const } c1 = str[j++]; - ++o; - + ++o; + if (j <= size) { - if (c == '\\' && patterStr[i] == '_' && c1 == '_') { i++; continue; } + if (c == '\\' && patterStr[i] == '_' && c1 == '_') { + i++; + continue; + } if (c == c1 || tolower(c) == tolower(c1) || (c == pInfo->matchOne && c1 != 0)) { continue; } @@ -365,21 +343,19 @@ int32_t WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t siz return TSDB_PATTERN_NOMATCH; } - + return (str[j] == 0 || j >= size) ? TSDB_PATTERN_MATCH : TSDB_PATTERN_NOMATCH; } -int32_t compareStrRegexCompMatch(const void* pLeft, const void* pRight) { - return compareStrRegexComp(pLeft, pRight); -} +int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight); } -int32_t compareStrRegexCompNMatch(const void* pLeft, const void* pRight) { +int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight) { return compareStrRegexComp(pLeft, pRight) ? 0 : 1; } -int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { +int32_t compareStrRegexComp(const void *pLeft, const void *pRight) { size_t sz = varDataLen(pRight); - char *pattern = malloc(sz + 1); + char *pattern = malloc(sz + 1); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); pattern[sz] = 0; @@ -414,15 +390,14 @@ int32_t compareStrRegexComp(const void* pLeft, const void* pRight) { return result; } -int32_t taosArrayCompareString(const void* a, const void* b) { - const char* x = *(const char**)a; - const char* y = *(const char**)b; +int32_t taosArrayCompareString(const void *a, const void *b) { + const char *x = *(const char **)a; + const char *y = *(const char **)b; return compareLenPrefixedStr(x, y); } - -int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { +int32_t compareStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN); @@ -430,7 +405,7 @@ int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); size_t sz = varDataLen(pLeft); - char *buf = malloc(sz + 1); + char *buf = malloc(sz + 1); memcpy(buf, varDataVal(pLeft), sz); buf[sz] = 0; @@ -440,11 +415,11 @@ int32_t compareStrPatternMatch(const void* pLeft, const void* pRight) { return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareStrPatternNotMatch(const void* pLeft, const void* pRight) { +int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight) { return compareStrPatternMatch(pLeft, pRight) ? 0 : 1; } -int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { +int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight) { SPatternCompareInfo pInfo = {'%', '_'}; assert(varDataLen(pRight) <= TSDB_MAX_FIELD_LEN * TSDB_NCHAR_SIZE); @@ -452,17 +427,16 @@ int32_t compareWStrPatternMatch(const void* pLeft, const void* pRight) { wchar_t *pattern = calloc(varDataLen(pRight) + 1, sizeof(wchar_t)); memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); - int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft)/TSDB_NCHAR_SIZE, &pInfo); + int32_t ret = WCSPatternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft) / TSDB_NCHAR_SIZE, &pInfo); free(pattern); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } -int32_t compareWStrPatternNotMatch(const void* pLeft, const void* pRight) { +int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight) { return compareWStrPatternMatch(pLeft, pRight) ? 0 : 1; } - __compar_fn_t getComparFunc(int32_t type, int32_t optr) { __compar_fn_t comparFn = NULL; @@ -514,13 +488,25 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { switch (type) { case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: comparFn = compareInt8Val; break; - case TSDB_DATA_TYPE_SMALLINT: comparFn = compareInt16Val; break; - case TSDB_DATA_TYPE_INT: comparFn = compareInt32Val; break; + case TSDB_DATA_TYPE_TINYINT: + comparFn = compareInt8Val; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = compareInt16Val; + break; + case TSDB_DATA_TYPE_INT: + comparFn = compareInt32Val; + break; case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: comparFn = compareInt64Val; break; - case TSDB_DATA_TYPE_FLOAT: comparFn = compareFloatVal; break; - case TSDB_DATA_TYPE_DOUBLE: comparFn = compareDoubleVal; break; + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = compareInt64Val; + break; + case TSDB_DATA_TYPE_FLOAT: + comparFn = compareFloatVal; + break; + case TSDB_DATA_TYPE_DOUBLE: + comparFn = compareDoubleVal; + break; case TSDB_DATA_TYPE_BINARY: { if (optr == OP_TYPE_MATCH) { comparFn = compareStrRegexCompMatch; @@ -560,10 +546,18 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { break; } - case TSDB_DATA_TYPE_UTINYINT: comparFn = compareUint8Val; break; - case TSDB_DATA_TYPE_USMALLINT: comparFn = compareUint16Val;break; - case TSDB_DATA_TYPE_UINT: comparFn = compareUint32Val;break; - case TSDB_DATA_TYPE_UBIGINT: comparFn = compareUint64Val;break; + case TSDB_DATA_TYPE_UTINYINT: + comparFn = compareUint8Val; + break; + case TSDB_DATA_TYPE_USMALLINT: + comparFn = compareUint16Val; + break; + case TSDB_DATA_TYPE_UINT: + comparFn = compareUint32Val; + break; + case TSDB_DATA_TYPE_UBIGINT: + comparFn = compareUint64Val; + break; default: comparFn = compareInt32Val; @@ -573,4 +567,106 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) { return comparFn; } +__compar_fn_t getKeyComparFunc(int32_t keyType, int32_t order) { + __compar_fn_t comparFn = NULL; + switch (keyType) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt8Val : compareInt8ValDesc; + break; + case TSDB_DATA_TYPE_SMALLINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt16Val : compareInt16ValDesc; + break; + case TSDB_DATA_TYPE_INT: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt64Val : compareInt64ValDesc; + break; + case TSDB_DATA_TYPE_FLOAT: + comparFn = (order == TSDB_ORDER_ASC) ? compareFloatVal : compareFloatValDesc; + break; + case TSDB_DATA_TYPE_DOUBLE: + comparFn = (order == TSDB_ORDER_ASC) ? compareDoubleVal : compareDoubleValDesc; + break; + case TSDB_DATA_TYPE_UTINYINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint8Val : compareUint8ValDesc; + break; + case TSDB_DATA_TYPE_USMALLINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint16Val : compareUint16ValDesc; + break; + case TSDB_DATA_TYPE_UINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint32Val : compareUint32ValDesc; + break; + case TSDB_DATA_TYPE_UBIGINT: + comparFn = (order == TSDB_ORDER_ASC) ? compareUint64Val : compareUint64ValDesc; + break; + case TSDB_DATA_TYPE_BINARY: + comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedStr : compareLenPrefixedStrDesc; + break; + case TSDB_DATA_TYPE_NCHAR: + comparFn = (order == TSDB_ORDER_ASC) ? compareLenPrefixedWStr : compareLenPrefixedWStrDesc; + break; + default: + comparFn = (order == TSDB_ORDER_ASC) ? compareInt32Val : compareInt32ValDesc; + break; + } + + return comparFn; +} + +int32_t doCompare(const char *f1, const char *f2, int32_t type, size_t size) { + switch (type) { + case TSDB_DATA_TYPE_INT: + DEFAULT_COMP(GET_INT32_VAL(f1), GET_INT32_VAL(f2)); + case TSDB_DATA_TYPE_DOUBLE: + DEFAULT_DOUBLE_COMP(GET_DOUBLE_VAL(f1), GET_DOUBLE_VAL(f2)); + case TSDB_DATA_TYPE_FLOAT: + DEFAULT_FLOAT_COMP(GET_FLOAT_VAL(f1), GET_FLOAT_VAL(f2)); + case TSDB_DATA_TYPE_BIGINT: + DEFAULT_COMP(GET_INT64_VAL(f1), GET_INT64_VAL(f2)); + case TSDB_DATA_TYPE_SMALLINT: + DEFAULT_COMP(GET_INT16_VAL(f1), GET_INT16_VAL(f2)); + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BOOL: + DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); + case TSDB_DATA_TYPE_UTINYINT: + DEFAULT_COMP(GET_UINT8_VAL(f1), GET_UINT8_VAL(f2)); + case TSDB_DATA_TYPE_USMALLINT: + DEFAULT_COMP(GET_UINT16_VAL(f1), GET_UINT16_VAL(f2)); + case TSDB_DATA_TYPE_UINT: + DEFAULT_COMP(GET_UINT32_VAL(f1), GET_UINT32_VAL(f2)); + case TSDB_DATA_TYPE_UBIGINT: + DEFAULT_COMP(GET_UINT64_VAL(f1), GET_UINT64_VAL(f2)); + case TSDB_DATA_TYPE_NCHAR: { + tstr *t1 = (tstr *)f1; + tstr *t2 = (tstr *)f2; + + if (t1->len != t2->len) { + return t1->len > t2->len ? 1 : -1; + } + int32_t ret = memcmp((wchar_t *)t1, (wchar_t *)t2, t2->len); + if (ret == 0) { + return ret; + } + return (ret < 0) ? -1 : 1; + } + default: { // todo refactor + tstr *t1 = (tstr *)f1; + tstr *t2 = (tstr *)f2; + + if (t1->len != t2->len) { + return t1->len > t2->len ? 1 : -1; + } else { + int32_t ret = strncmp(t1->data, t2->data, t1->len); + if (ret == 0) { + return 0; + } else { + return ret < 0 ? -1 : 1; + } + } + } + } +} diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b0dcff88e7e0b946b9118094aa08d4a4ec04ae8d..b2332aded7fd79beb8feb6539ddf8a1200de4a5b 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -16,7 +16,7 @@ /* README.md TAOS compression * * INTEGER Compression Algorithm: - * To compress integers (including char, short, int, int64_t), the difference + * To compress integers (including char, short, int32_t, int64_t), the difference * between two integers is calculated at first. Then the difference is * transformed to positive by zig-zag encoding method * (https://gist.github.com/mfuerstenau/ba870a29e16536fdbaba). Then the value is @@ -47,69 +47,65 @@ * */ -#include "os.h" +#define _DEFAULT_SOURCE +#include "tcompression.h" #include "lz4.h" -#ifdef TD_TSZ - #include "td_sz.h" +#include "tlog.h" + +#ifdef TD_TSZ +#include "td_sz.h" #endif -#include "tcompression.h" -#include "ulog.h" -static const int TEST_NUMBER = 1; -#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) +static const int32_t TEST_NUMBER = 1; +#define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) #define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L) -#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) +#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode #define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode #ifdef TD_TSZ -bool lossyFloat = false; +bool lossyFloat = false; bool lossyDouble = false; // init call -int tsCompressInit(){ - // config - if(lossyColumns[0] == 0){ +int32_t tsCompressInit() { + // config + if (lossyColumns[0] == 0) { lossyFloat = false; lossyDouble = false; return 0; } - lossyFloat = strstr(lossyColumns, "float") != NULL; + lossyFloat = strstr(lossyColumns, "float") != NULL; lossyDouble = strstr(lossyColumns, "double") != NULL; - if(lossyFloat == false && lossyDouble == false) - return 0; - + if (lossyFloat == false && lossyDouble == false) return 0; + tdszInit(fPrecision, dPrecision, maxRange, curRange, Compressor); - if(lossyFloat) - uInfo("lossy compression float is opened. "); - if(lossyDouble) - uInfo("lossy compression double is opened. "); + if (lossyFloat) uInfo("lossy compression float is opened. "); + if (lossyDouble) uInfo("lossy compression double is opened. "); return 1; } // exit call -void tsCompressExit(){ - tdszExit(); -} +void tsCompressExit() { tdszExit(); } #endif /* * Compress Integer (Simple8B). */ -int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type) { +int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 // 12 13 14 15 - char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; - int selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; - char bit_to_selector[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, + char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + char bit_to_selector[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 12, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; // get the byte limit. - int word_length = 0; + int32_t word_length = 0; switch (type) { case TSDB_DATA_TYPE_BIGINT: word_length = LONG_BYTES; @@ -128,17 +124,17 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o return -1; } - int byte_limit = nelements * word_length + 1; - int opos = 1; + int32_t byte_limit = nelements * word_length + 1; + int32_t opos = 1; int64_t prev_value = 0; - for (int i = 0; i < nelements;) { + for (int32_t i = 0; i < nelements;) { char selector = 0; char bit = 0; - int elems = 0; + int32_t elems = 0; int64_t prev_value_tmp = prev_value; - for (int j = i; j < nelements; j++) { + for (int32_t j = i; j < nelements; j++) { // Read data from the input stream and convert it to INT64 type. int64_t curr_value = 0; switch (type) { @@ -172,16 +168,17 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o tmp_bit = (LONG_BYTES * BITS_PER_BYTE) - BUILDIN_CLZL(zigzag_value); } - if (elems + 1 <= selector_to_elems[(int)selector] && elems + 1 <= selector_to_elems[(int)(bit_to_selector[(int)tmp_bit])]) { + if (elems + 1 <= selector_to_elems[(int32_t)selector] && + elems + 1 <= selector_to_elems[(int32_t)(bit_to_selector[(int32_t)tmp_bit])]) { // If can hold another one. - selector = selector > bit_to_selector[(int)tmp_bit] ? selector : bit_to_selector[(int)tmp_bit]; + selector = selector > bit_to_selector[(int32_t)tmp_bit] ? selector : bit_to_selector[(int32_t)tmp_bit]; elems++; - bit = bit_per_integer[(int)selector]; + bit = bit_per_integer[(int32_t)selector]; } else { // if cannot hold another one. - while (elems < selector_to_elems[(int)selector]) selector++; - elems = selector_to_elems[(int)selector]; - bit = bit_per_integer[(int)selector]; + while (elems < selector_to_elems[(int32_t)selector]) selector++; + elems = selector_to_elems[(int32_t)selector]; + bit = bit_per_integer[(int32_t)selector]; break; } prev_value_tmp = curr_value; @@ -189,7 +186,7 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o uint64_t buffer = 0; buffer |= (uint64_t)selector; - for (int k = 0; k < elems; k++) { + for (int32_t k = 0; k < elems; k++) { int64_t curr_value = 0; /* get current values */ switch (type) { case TSDB_DATA_TYPE_TINYINT: @@ -229,8 +226,8 @@ int tsCompressINTImp(const char *const input, const int nelements, char *const o return opos; } -int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type) { - int word_length = 0; +int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { + int32_t word_length = 0; switch (type) { case TSDB_DATA_TYPE_BIGINT: word_length = LONG_BYTES; @@ -257,12 +254,12 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 // 12 13 14 15 - char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; - int selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; const char *ip = input + 1; - int count = 0; - int _pos = 0; + int32_t count = 0; + int32_t _pos = 0; int64_t prev_value = 0; while (1) { @@ -271,11 +268,11 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const uint64_t w = 0; memcpy(&w, ip, LONG_BYTES); - char selector = (char)(w & INT64MASK(4)); // selector = 4 - char bit = bit_per_integer[(int)selector]; // bit = 3 - int elems = selector_to_elems[(int)selector]; + char selector = (char)(w & INT64MASK(4)); // selector = 4 + char bit = bit_per_integer[(int32_t)selector]; // bit = 3 + int32_t elems = selector_to_elems[(int32_t)selector]; - for (int i = 0; i < elems; i++) { + for (int32_t i = 0; i < elems; i++) { uint64_t zigzag_value; if (selector == 0 || selector == 1) { @@ -320,11 +317,11 @@ int tsDecompressINTImp(const char *const input, const int nelements, char *const /* ----------------------------------------------Bool Compression * ---------------------------------------------- */ // TODO: You can also implement it using RLE method. -int tsCompressBoolImp(const char *const input, const int nelements, char *const output) { - int pos = -1; - int ele_per_byte = BITS_PER_BYTE / 2; +int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output) { + int32_t pos = -1; + int32_t ele_per_byte = BITS_PER_BYTE / 2; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % ele_per_byte == 0) { pos++; output[pos] = 0; @@ -351,11 +348,11 @@ int tsCompressBoolImp(const char *const input, const int nelements, char *const return pos + 1; } -int tsDecompressBoolImp(const char *const input, const int nelements, char *const output) { - int ipos = -1, opos = 0; - int ele_per_byte = BITS_PER_BYTE / 2; +int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output) { + int32_t ipos = -1, opos = 0; + int32_t ele_per_byte = BITS_PER_BYTE / 2; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % ele_per_byte == 0) { ipos++; } @@ -374,10 +371,10 @@ int tsDecompressBoolImp(const char *const input, const int nelements, char *cons } /* Run Length Encoding(RLE) Method */ -int tsCompressBoolRLEImp(const char *const input, const int nelements, char *const output) { - int _pos = 0; +int32_t tsCompressBoolRLEImp(const char *const input, const int32_t nelements, char *const output) { + int32_t _pos = 0; - for (int i = 0; i < nelements;) { + for (int32_t i = 0; i < nelements;) { unsigned char counter = 1; char num = input[i]; @@ -407,8 +404,8 @@ int tsCompressBoolRLEImp(const char *const input, const int nelements, char *con return _pos; } -int tsDecompressBoolRLEImp(const char *const input, const int nelements, char *const output) { - int ipos = 0, opos = 0; +int32_t tsDecompressBoolRLEImp(const char *const input, const int32_t nelements, char *const output) { + int32_t ipos = 0, opos = 0; while (1) { char encode = input[ipos++]; unsigned counter = (encode >> 1) & INT8MASK(7); @@ -427,9 +424,9 @@ int tsDecompressBoolRLEImp(const char *const input, const int nelements, char *c // Note: the size of the output must be larger than input_size + 1 and // LZ4_compressBound(size) + 1; // >= max(input_size, LZ4_compressBound(input_size)) + 1; -int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize) { +int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize) { // Try to compress using LZ4 algorithm. - const int compressed_data_size = LZ4_compress_default(input, output + 1, inputSize, outputSize-1); + const int32_t compressed_data_size = LZ4_compress_default(input, output + 1, inputSize, outputSize - 1); // If cannot compress or after compression, data becomes larger. if (compressed_data_size <= 0 || compressed_data_size > inputSize) { @@ -443,12 +440,12 @@ int tsCompressStringImp(const char *const input, int inputSize, char *const outp return compressed_data_size + 1; } -int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize) { +int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, int32_t outputSize) { // compressedSize is the size of data after compression. - + if (input[0] == 1) { /* It is compressed by LZ4 algorithm */ - const int decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); + const int32_t decompressed_size = LZ4_decompress_safe(input + 1, output, compressedSize - 1, outputSize); if (decompressed_size < 0) { uError("Failed to decompress string with LZ4 algorithm, decompressed size:%d", decompressed_size); return -1; @@ -468,24 +465,24 @@ int tsDecompressStringImp(const char *const input, int compressedSize, char *con /* --------------------------------------------Timestamp Compression * ---------------------------------------------- */ // TODO: Take care here, we assumes little endian encoding. -int tsCompressTimestampImp(const char *const input, const int nelements, char *const output) { - int _pos = 1; +int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { + int32_t _pos = 1; assert(nelements >= 0); if (nelements == 0) return 0; int64_t *istream = (int64_t *)input; - int64_t prev_value = istream[0]; - if(prev_value >= 0x8000000000000000) { - uWarn("compression timestamp is over signed long long range. ts = 0x%"PRIx64" \n", prev_value); - goto _exit_over; + int64_t prev_value = istream[0]; + if (prev_value >= 0x8000000000000000) { + uWarn("compression timestamp is over signed long long range. ts = 0x%" PRIx64 " \n", prev_value); + goto _exit_over; } int64_t prev_delta = -prev_value; uint8_t flags = 0, flag1 = 0, flag2 = 0; uint64_t dd1 = 0, dd2 = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { int64_t curr_value = istream[i]; if (!safeInt64Add(curr_value, -prev_value)) goto _exit_over; int64_t curr_delta = curr_value - prev_value; @@ -564,7 +561,7 @@ _exit_over: return nelements * LONG_BYTES + 1; } -int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { assert(nelements >= 0); if (nelements == 0) return 0; @@ -574,7 +571,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char } else if (input[0] == 1) { // Decompress int64_t *ostream = (int64_t *)output; - int ipos = 1, opos = 0; + int32_t ipos = 1, opos = 0; int8_t nbytes = 0; int64_t prev_value = 0; int64_t prev_delta = 0; @@ -635,9 +632,9 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char } /* --------------------------------------------Double Compression * ---------------------------------------------- */ -void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int *const pos) { +void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int32_t *const pos) { uint8_t nbytes = (flag & INT8MASK(3)) + 1; - int nshift = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t nshift = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff >>= nshift; while (nbytes) { @@ -647,9 +644,9 @@ void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int *con } } -int tsCompressDoubleImp(const char *const input, const int nelements, char *const output) { - int byte_limit = nelements * DOUBLE_BYTES + 1; - int opos = 1; +int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output) { + int32_t byte_limit = nelements * DOUBLE_BYTES + 1; + int32_t opos = 1; uint64_t prev_value = 0; uint64_t prev_diff = 0; @@ -658,7 +655,7 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons double *istream = (double *)input; // Main loop - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { union { double real; uint64_t bits; @@ -670,8 +667,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons uint64_t predicted = prev_value; uint64_t diff = curr.bits ^ predicted; - int leading_zeros = LONG_BYTES * BITS_PER_BYTE; - int trailing_zeros = leading_zeros; + int32_t leading_zeros = LONG_BYTES * BITS_PER_BYTE; + int32_t trailing_zeros = leading_zeros; if (diff) { trailing_zeros = BUILDIN_CTZL(diff); @@ -696,8 +693,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons prev_diff = diff; prev_flag = flag; } else { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = (flag & INT8MASK(3)) + 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = (flag & INT8MASK(3)) + 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag | (flag << 4); output[opos++] = flags; @@ -713,8 +710,8 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons } if (nelements % 2) { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag; output[opos++] = flags; @@ -731,19 +728,19 @@ int tsCompressDoubleImp(const char *const input, const int nelements, char *cons return opos; } -uint64_t decodeDoubleValue(const char *const input, int *const ipos, uint8_t flag) { +uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint64_t diff = 0ul; - int nbytes = (flag & INT8MASK(3)) + 1; - for (int i = 0; i < nbytes; i++) { + int32_t nbytes = (flag & INT8MASK(3)) + 1; + for (int32_t i = 0; i < nbytes; i++) { diff = diff | ((INT64MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); } - int shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; return diff; } -int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output) { // output stream double *ostream = (double *)output; @@ -753,11 +750,11 @@ int tsDecompressDoubleImp(const char *const input, const int nelements, char *co } uint8_t flags = 0; - int ipos = 1; - int opos = 0; + int32_t ipos = 1; + int32_t opos = 0; uint64_t prev_value = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % 2 == 0) { flags = input[ipos++]; } @@ -783,9 +780,9 @@ int tsDecompressDoubleImp(const char *const input, const int nelements, char *co /* --------------------------------------------Float Compression * ---------------------------------------------- */ -void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int *const pos) { +void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int32_t *const pos) { uint8_t nbytes = (flag & INT8MASK(3)) + 1; - int nshift = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t nshift = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff >>= nshift; while (nbytes) { @@ -795,17 +792,17 @@ void encodeFloatValue(uint32_t diff, uint8_t flag, char *const output, int *cons } } -int tsCompressFloatImp(const char *const input, const int nelements, char *const output) { - float *istream = (float *)input; - int byte_limit = nelements * FLOAT_BYTES + 1; - int opos = 1; +int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output) { + float *istream = (float *)input; + int32_t byte_limit = nelements * FLOAT_BYTES + 1; + int32_t opos = 1; uint32_t prev_value = 0; uint32_t prev_diff = 0; uint8_t prev_flag = 0; // Main loop - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { union { float real; uint32_t bits; @@ -817,8 +814,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const uint32_t predicted = prev_value; uint32_t diff = curr.bits ^ predicted; - int leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; - int trailing_zeros = leading_zeros; + int32_t leading_zeros = FLOAT_BYTES * BITS_PER_BYTE; + int32_t trailing_zeros = leading_zeros; if (diff) { trailing_zeros = BUILDIN_CTZ(diff); @@ -843,8 +840,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const prev_diff = diff; prev_flag = flag; } else { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = (flag & INT8MASK(3)) + 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = (flag & INT8MASK(3)) + 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag | (flag << 4); output[opos++] = flags; @@ -860,8 +857,8 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const } if (nelements % 2) { - int nbyte1 = (prev_flag & INT8MASK(3)) + 1; - int nbyte2 = 1; + int32_t nbyte1 = (prev_flag & INT8MASK(3)) + 1; + int32_t nbyte2 = 1; if (opos + 1 + nbyte1 + nbyte2 <= byte_limit) { uint8_t flags = prev_flag; output[opos++] = flags; @@ -878,19 +875,19 @@ int tsCompressFloatImp(const char *const input, const int nelements, char *const return opos; } -uint32_t decodeFloatValue(const char *const input, int *const ipos, uint8_t flag) { +uint32_t decodeFloatValue(const char *const input, int32_t *const ipos, uint8_t flag) { uint32_t diff = 0ul; - int nbytes = (flag & INT8MASK(3)) + 1; - for (int i = 0; i < nbytes; i++) { + int32_t nbytes = (flag & INT8MASK(3)) + 1; + for (int32_t i = 0; i < nbytes; i++) { diff = diff | ((INT32MASK(8) & input[(*ipos)++]) << BITS_PER_BYTE * i); } - int shift_width = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t shift_width = (FLOAT_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; return diff; } -int tsDecompressFloatImp(const char *const input, const int nelements, char *const output) { +int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output) { float *ostream = (float *)output; if (input[0] == 1) { @@ -899,11 +896,11 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con } uint8_t flags = 0; - int ipos = 1; - int opos = 0; + int32_t ipos = 1; + int32_t opos = 0; uint32_t prev_value = 0; - for (int i = 0; i < nelements; i++) { + for (int32_t i = 0; i < nelements; i++) { if (i % 2 == 0) { flags = input[ipos++]; } @@ -927,15 +924,15 @@ int tsDecompressFloatImp(const char *const input, const int nelements, char *con return nelements * FLOAT_BYTES; } -#ifdef TD_TSZ +#ifdef TD_TSZ // // ---------- float double lossy ----------- // -int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output){ - // compress with sz - int compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1); +int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output) { + // compress with sz + int32_t compressedSize = tdszCompress(SZ_FLOAT, input, nelements, output + 1); unsigned char algo = ALGO_SZ_LOSSY << 1; - if (compressedSize == 0 || compressedSize >= nelements*sizeof(float)){ + if (compressedSize == 0 || compressedSize >= nelements * sizeof(float)) { // compressed error or large than original output[0] = MODE_NOCOMPRESS | algo; memcpy(output + 1, input, nelements * sizeof(float)); @@ -949,25 +946,26 @@ int tsCompressFloatLossyImp(const char * input, const int nelements, char *const return compressedSize; } -int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ - int decompressedSize = 0; - if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ +int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output) { + int32_t decompressedSize = 0; + if (HEAD_MODE(input[0]) == MODE_NOCOMPRESS) { // orginal so memcpy directly decompressedSize = nelements * sizeof(float); memcpy(output, input + 1, decompressedSize); return decompressedSize; - } + } // decompressed with sz return tdszDecompress(SZ_FLOAT, input + 1, compressedSize - 1, nelements, output); } -int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output){ - // compress with sz - int compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1); +int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output) { + // compress with sz + int32_t compressedSize = tdszCompress(SZ_DOUBLE, input, nelements, output + 1); unsigned char algo = ALGO_SZ_LOSSY << 1; - if (compressedSize == 0 || compressedSize >= nelements*sizeof(double)) { + if (compressedSize == 0 || compressedSize >= nelements * sizeof(double)) { // compressed error or large than original output[0] = MODE_NOCOMPRESS | algo; memcpy(output + 1, input, nelements * sizeof(double)); @@ -978,18 +976,19 @@ int tsCompressDoubleLossyImp(const char * input, const int nelements, char *cons compressedSize += 1; } - return compressedSize; + return compressedSize; } -int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output){ - int decompressedSize = 0; - if( HEAD_MODE(input[0]) == MODE_NOCOMPRESS){ +int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, + char *const output) { + int32_t decompressedSize = 0; + if (HEAD_MODE(input[0]) == MODE_NOCOMPRESS) { // orginal so memcpy directly decompressedSize = nelements * sizeof(double); memcpy(output, input + 1, decompressedSize); return decompressedSize; - } + } // decompressed with sz return tdszDecompress(SZ_DOUBLE, input + 1, compressedSize - 1, nelements, output); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 469da11d9312b6357711ac05777a0ea1d933b605..8330c10ec703b7c2484167ff5cba7d46b9eabe85 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -14,494 +14,647 @@ */ #define _DEFAULT_SOURCE -#include "os.h" #include "tconfig.h" +#include "taoserror.h" +#include "tlog.h" #include "tutil.h" -#include "ulog.h" - -SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}}; -int32_t tsGlobalConfigNum = 0; - -static char *tsGlobalUnit[] = { - " ", - "(%)", - "(GB)", - "(Mb)", - "(byte)", - "(s)", - "(ms)" -}; - -char *tsCfgStatusStr[] = { - "none", - "system default", - "config file", - "taos_options", - "program argument list" -}; - -static void taosReadFloatConfig(SGlobalCfg *cfg, char *input_value) { - float value = (float)atof(input_value); - float *option = (float *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } + +#define CFG_NAME_PRINT_LEN 24 +#define CFG_SRC_PRINT_LEN 12 + +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); +int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); +int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); + +SConfig *cfgInit() { + SConfig *pCfg = calloc(1, sizeof(SConfig)); + if (pCfg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } -} -static void taosReadDoubleConfig(SGlobalCfg *cfg, char *input_value) { - double value = atof(input_value); - double *option = (double *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%f", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %f", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } + pCfg->array = taosArrayInit(32, sizeof(SConfigItem)); + if (pCfg->array == NULL) { + free(pCfg); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } + + return pCfg; } -static void taosReadInt32Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int32_t *option = (int32_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { + switch (cfgType) { + case CFG_STYPE_CFG_FILE: + return cfgLoadFromCfgFile(pCfg, sourceStr); + case CFG_STYPE_ENV_FILE: + return cfgLoadFromEnvFile(pCfg, sourceStr); + case CFG_STYPE_ENV_VAR: + return cfgLoadFromEnvVar(pCfg); + case CFG_STYPE_APOLLO_URL: + return cfgLoadFromApollUrl(pCfg, sourceStr); + default: + return -1; } } -static void taosReadInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int16_t *option = (int16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); +int32_t cfgLoadArray(SConfig *pCfg, SArray *pArgs) { + int32_t size = taosArrayGetSize(pArgs); + for (int32_t i = 0; i < size; ++i) { + SConfigPair *pPair = taosArrayGet(pArgs, i); + if (cfgSetItem(pCfg, pPair->name, pPair->value, CFG_STYPE_ARG_LIST) != 0) { + return -1; } } + + return 0; } -static void taosReadUInt16Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - uint16_t *option = (uint16_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (uint16_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); - } +static void cfgFreeItem(SConfigItem *pItem) { + if (pItem->dtype == CFG_DTYPE_STRING || pItem->dtype == CFG_DTYPE_DIR || pItem->dtype == CFG_DTYPE_LOCALE || + pItem->dtype == CFG_DTYPE_CHARSET || pItem->dtype == CFG_DTYPE_TIMEZONE) { + tfree(pItem->str); + } + if (pItem->array) { + taosArrayDestroy(pItem->array); + pItem->array = NULL; } } -static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) { - int32_t value = atoi(input_value); - int8_t *option = (int8_t *)cfg->ptr; - if (value < cfg->minValue || value > cfg->maxValue) { - uError("config option:%s, input value:%s, out of range[%f, %f], use default value:%d", - cfg->option, input_value, cfg->minValue, cfg->maxValue, *option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - *option = (int8_t)value; - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %d", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], *option); +void cfgCleanup(SConfig *pCfg) { + if (pCfg != NULL) { + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + cfgFreeItem(pItem); + tfree(pItem->name); } + taosArrayDestroy(pCfg->array); + free(pCfg); } } -static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int)strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option, - input_value, cfg->ptrLength, option); - return false; - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - taosExpandDir(input_value, option, cfg->ptrLength); - taosRealPath(option, cfg->ptrLength); - - if (taosMkDir(option) != 0) { - uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value, - strerror(errno)); - return false; - } - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } +int32_t cfgGetSize(SConfig *pCfg) { return taosArrayGetSize(pCfg->array); } + +static int32_t cfgCheckAndSetTimezone(SConfigItem *pItem, const char *timezone) { + cfgFreeItem(pItem); + pItem->str = strdup(timezone); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } - return true; + return 0; } -static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = taosInetAddr(input_value); - char * option = (char *)cfg->ptr; - if (value == INADDR_NONE) { - uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", - cfg->option, input_value, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } +static int32_t cfgCheckAndSetCharset(SConfigItem *pItem, const char *charset) { + cfgFreeItem(pItem); + pItem->str = strdup(charset); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + + return 0; } -static void taosReadStringConfig(SGlobalCfg *cfg, char *input_value) { - int length = (int) strlen(input_value); - char *option = (char *)cfg->ptr; - if (length <= 0 || length > cfg->ptrLength) { - uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", - cfg->option, input_value, cfg->ptrLength, option); - } else { - if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) { - strncpy(option, input_value, cfg->ptrLength); - cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE; - } else { - uWarn("config option:%s, input value:%s, is configured by %s, use %s", cfg->option, input_value, - tsCfgStatusStr[cfg->cfgStatus], option); - } +static int32_t cfgCheckAndSetLocale(SConfigItem *pItem, const char *locale) { + cfgFreeItem(pItem); + pItem->str = strdup(locale); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; } + + return 0; } -static void taosReadLogOption(char *option, char *value) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG) || !(cfg->cfgType & TSDB_CFG_CTYPE_B_LOG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; +static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { + char fullDir[PATH_MAX] = {0}; + if (taosExpandDir(inputDir, fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to expand dir:%s since %s", inputDir, terrstr()); + return -1; + } - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - // if (strcasecmp(cfg->option, "debugFlag") == 0) { - // taosSetAllDebugFlag(); - // } - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - default: - break; - } - break; + if (taosRealPath(fullDir, PATH_MAX) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); + return -1; } + + if (taosMkDir(fullDir) != 0) { + uError("failed to create dir:%s realpath:%s since %s", inputDir, fullDir, terrstr()); + return -1; + } + + cfgFreeItem(pItem); + pItem->str = strdup(fullDir); + if (pItem->str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; } -SGlobalCfg *taosGetConfigOption(const char *option) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (strcasecmp(cfg->option, option) != 0) continue; - return cfg; +static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + bool tmp = false; + if (strcasecmp(value, "true") == 0) { + tmp = true; } - return NULL; + if (atoi(value) > 0) { + tmp = true; + } + + pItem->bval = tmp; + pItem->stype = stype; + return 0; } -static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) { - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue; - if (strcasecmp(cfg->option, option) != 0) continue; +static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int32_t ival = (int32_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "], use last src:%s value:%d", + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i32); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - taosReadInt8Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT16: - taosReadInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_INT32: - taosReadInt32Config(cfg, value); - break; - case TAOS_CFG_VTYPE_UINT16: - taosReadUInt16Config(cfg, value); - break; - case TAOS_CFG_VTYPE_FLOAT: - taosReadFloatConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DOUBLE: - taosReadDoubleConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_STRING: - taosReadStringConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_IPSTR: - taosReadIpStrConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DIRECTORY: - taosReadDirectoryConfig(cfg, value); - break; - case TAOS_CFG_VTYPE_DATA_DIRCTORY: - if (taosReadDirectoryConfig(cfg, value)) { - // taosReadDataDirCfg(value, value2, value3); - } - break; - default: - uError("config option:%s, input value:%s, can't be recognized", option, value); - break; - } - break; + pItem->i32 = ival; + pItem->stype = stype; + return 0; +} + +static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + int64_t ival = (int64_t)atoi(value); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 + "], use last src:%s value:%" PRId64, + pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax, + cfgStypeStr(pItem->stype), pItem->i64); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; } + + pItem->i64 = ival; + pItem->stype = stype; + return 0; } -void taosAddConfigOption(SGlobalCfg cfg) { - tsGlobalConfig[tsGlobalConfigNum++] = cfg; +static int32_t cfgSetFloat(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + float fval = (float)atof(value); + if (fval < pItem->fmin || fval > pItem->fmax) { + uError("cfg:%s, type:%s src:%s value:%f out of range[%f, %f], use last src:%s value:%f", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), fval, pItem->fmin, pItem->fmax, cfgStypeStr(pItem->stype), + pItem->fval); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + + pItem->fval = fval; + pItem->stype = stype; + return 0; } -void taosReadGlobalLogCfg() { - FILE * fp; - char * line, *option, *value; - int olen, vlen; - char fileName[PATH_MAX] = {0}; +static int32_t cfgSetString(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + char *tmp = strdup(value); + if (tmp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + free(pItem->str); + pItem->str = tmp; + pItem->stype = stype; + return 0; +} - taosExpandDir(configDir, configDir, PATH_MAX); - taosReadLogOption("logDir", tsLogDir); - - sprintf(fileName, "%s/taos.cfg", configDir); - fp = fopen(fileName, "r"); - if (fp == NULL) { - printf("\nconfig file:%s not found, all variables are set to default\n", fileName); - return; +static int32_t cfgSetDir(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (cfgCheckAndSetDir(pItem, value) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; } - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); - - option = value = NULL; - olen = vlen = 0; - - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { - break; - } + pItem->stype = stype; + return 0; +} - line[len - 1] = 0; +static int32_t cfgSetLocale(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (cfgCheckAndSetLocale(pItem, value) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; + pItem->stype = stype; + return 0; +} - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; +static int32_t cfgSetCharset(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (cfgCheckAndSetCharset(pItem, value) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; + } + + pItem->stype = stype; + return 0; +} - taosReadLogOption(option, value); +static int32_t cfgSetTimezone(SConfigItem *pItem, const char *value, ECfgSrcType stype) { + if (cfgCheckAndSetTimezone(pItem, value) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + uError("cfg:%s, type:%s src:%s value:%s failed to dup since %s, use last src:%s value:%s", pItem->name, + cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), value, terrstr(), cfgStypeStr(pItem->stype), pItem->str); + return -1; } - tfree(line); - fclose(fp); + pItem->stype = stype; + return 0; } -int32_t taosReadCfgFromFile() { - char * line, *option, *value, *value2, *value3; - int olen, vlen, vlen2, vlen3; - char fileName[PATH_MAX] = {0}; +static int32_t cfgSetTfsItem(SConfig *pCfg, const char *name, const char *value, const char *level, const char *primary, + ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) return -1; - sprintf(fileName, "%s/taos.cfg", configDir); - FILE *fp = fopen(fileName, "r"); - if (fp == NULL) { - fp = fopen(configDir, "r"); - if (fp == NULL) { + if (pItem->array == NULL) { + pItem->array = taosArrayInit(16, sizeof(SDiskCfg)); + if (pItem->array == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } - ssize_t _bytes = 0; - size_t len = 1024; - line = calloc(1, len); - - while (!feof(fp)) { - memset(line, 0, len); + SDiskCfg cfg = {0}; + tstrncpy(cfg.dir, value, sizeof(cfg.dir)); + cfg.level = atoi(level); + cfg.primary = atoi(primary); + void *ret = taosArrayPush(pItem->array, &cfg); + if (ret == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - option = value = value2 = value3 = NULL; - olen = vlen = vlen2 = vlen3 = 0; + pItem->stype = stype; + return 0; +} - _bytes = tgetline(&line, &len, fp); - if (_bytes < 0) - { +int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype) { + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (pItem == NULL) { + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + return cfgSetBool(pItem, value, stype); + case CFG_DTYPE_INT32: + return cfgSetInt32(pItem, value, stype); + case CFG_DTYPE_INT64: + return cfgSetInt64(pItem, value, stype); + case CFG_DTYPE_FLOAT: + return cfgSetFloat(pItem, value, stype); + case CFG_DTYPE_STRING: + return cfgSetString(pItem, value, stype); + case CFG_DTYPE_DIR: + return cfgSetDir(pItem, value, stype); + case CFG_DTYPE_TIMEZONE: + return cfgSetTimezone(pItem, value, stype); + case CFG_DTYPE_CHARSET: + return cfgSetCharset(pItem, value, stype); + case CFG_DTYPE_LOCALE: + return cfgSetLocale(pItem, value, stype); + case CFG_DTYPE_NONE: + default: break; + } + + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + if (strcasecmp(pItem->name, name) == 0) { + return pItem; } + } - line[len - 1] = 0; - - paGetToken(line, &option, &olen); - if (olen == 0) continue; - option[olen] = 0; + terrno = TSDB_CODE_CFG_NOT_FOUND; + return NULL; +} - paGetToken(option + olen + 1, &value, &vlen); - if (vlen == 0) continue; - value[vlen] = 0; +static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { + pItem->stype = CFG_STYPE_DEFAULT; + pItem->name = strdup(name); + if (pItem->name == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - paGetToken(value + vlen + 1, &value2, &vlen2); - if (vlen2 != 0) { - value2[vlen2] = 0; - paGetToken(value2 + vlen2 + 1, &value3, &vlen3); - if (vlen3 != 0) value3[vlen3] = 0; + int32_t len = strlen(name); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + + if (taosArrayPush(pCfg->array, pItem) == NULL) { + if (pItem->dtype == CFG_DTYPE_STRING) { + free(pItem->str); } + free(pItem->name); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} - taosReadConfigOption(option, value, value2, value3); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; } - fclose(fp); + SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } - tfree(line); + SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); +} - // if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { - // taosSetAllDebugFlag(); - // } +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, double minval, double maxval, bool tsc) { + if (defaultVal < minval || defaultVal > maxval) { + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } - return 0; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .tsc = tsc}; + return cfgAddItem(pCfg, &item, name); } -void taosPrintCfg() { - uInfo(" taos config & system info:"); - uInfo("=================================="); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .tsc = tsc}; + item.str = strdup(defaultVal); + if (item.str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + return cfgAddItem(pCfg, &item, name); +} - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, bool tsc) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .tsc = tsc}; + if (cfgCheckAndSetDir(&item, defaultVal) != 0) { + return -1; + } - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; + return cfgAddItem(pCfg, &item, name); +} - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - uInfo(" %s:%s%d%s", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_UINT16: - uInfo(" %s:%s%d%s", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_FLOAT: - uInfo(" %s:%s%f%s", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_DOUBLE: - uInfo(" %s:%s%f%s", cfg->option, blank, *((double *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - uInfo(" %s:%s%s%s", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); - break; - default: - break; - } +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .tsc = 1}; + if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { + return -1; } - taosPrintOsInfo(); - uInfo("=================================="); + return cfgAddItem(pCfg, &item, name); } -static void taosDumpCfg(SGlobalCfg *cfg) { - int optionLen = (int)strlen(cfg->option); - int blankLen = TSDB_CFG_PRINT_LEN - optionLen; - blankLen = blankLen < 0 ? 0 : blankLen; +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .tsc = 1}; + if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { + return -1; + } - char blank[TSDB_CFG_PRINT_LEN]; - memset(blank, ' ', TSDB_CFG_PRINT_LEN); - blank[blankLen] = 0; + return cfgAddItem(pCfg, &item, name); +} + +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .tsc = 1}; + if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { + return -1; + } + + return cfgAddItem(pCfg, &item, name); +} + +const char *cfgStypeStr(ECfgSrcType type) { + switch (type) { + case CFG_STYPE_DEFAULT: + return "default"; + case CFG_STYPE_CFG_FILE: + return "cfg_file"; + case CFG_STYPE_ENV_FILE: + return "env_file"; + case CFG_STYPE_ENV_VAR: + return "env_var"; + case CFG_STYPE_APOLLO_URL: + return "apollo_url"; + case CFG_STYPE_ARG_LIST: + return "arg_list"; + default: + return "invalid"; + } +} + +const char *cfgDtypeStr(ECfgDataType type) { + switch (type) { + case CFG_DTYPE_NONE: + return "none"; + case CFG_DTYPE_BOOL: + return "bool"; + case CFG_DTYPE_INT32: + return "int32"; + case CFG_DTYPE_INT64: + return "int64"; + case CFG_DTYPE_FLOAT: + return "float"; + case CFG_DTYPE_STRING: + return "string"; + case CFG_DTYPE_DIR: + return "dir"; + case CFG_DTYPE_LOCALE: + return "locale"; + case CFG_DTYPE_CHARSET: + return "charset"; + case CFG_DTYPE_TIMEZONE: + return "timezone"; + default: + return "invalid"; + } +} + +void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { + if (dump) { + printf(" global config"); + printf("\n"); + printf("================================================================="); + printf("\n"); + } else { + uInfo(" global config"); + uInfo("================================================================="); + } + + char src[CFG_SRC_PRINT_LEN + 1] = {0}; + char name[CFG_NAME_PRINT_LEN + 1] = {0}; + + int32_t size = taosArrayGetSize(pCfg->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = taosArrayGet(pCfg->array, i); + if (tsc && !pItem->tsc) continue; + tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); + for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { + if (src[i] == 0) src[i] = ' '; + } + + tstrncpy(name, pItem->name, CFG_NAME_PRINT_LEN); + for (int32_t i = 0; i < CFG_NAME_PRINT_LEN; ++i) { + if (name[i] == 0) name[i] = ' '; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + if (dump) { + printf("%s %s %u", src, name, pItem->bval); + printf("\n"); + } else { + uInfo("%s %s %u", src, name, pItem->bval); + } - switch (cfg->valType) { - case TAOS_CFG_VTYPE_INT8: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int8_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); - break; - case TAOS_CFG_VTYPE_INT32: - printf(" %s:%s%d%s\n", cfg->option, blank, *((int32_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); break; - case TAOS_CFG_VTYPE_UINT16: - printf(" %s:%s%d%s\n", cfg->option, blank, *((uint16_t *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + case CFG_DTYPE_INT32: + if (dump) { + printf("%s %s %d", src, name, pItem->i32); + printf("\n"); + } else { + uInfo("%s %s %d", src, name, pItem->i32); + } break; - case TAOS_CFG_VTYPE_FLOAT: - printf(" %s:%s%f%s\n", cfg->option, blank, *((float *)cfg->ptr), tsGlobalUnit[cfg->unitType]); + case CFG_DTYPE_INT64: + if (dump) { + printf("%s %s %" PRId64, src, name, pItem->i64); + printf("\n"); + } else { + uInfo("%s %s %" PRId64, src, name, pItem->i64); + } break; - case TAOS_CFG_VTYPE_STRING: - case TAOS_CFG_VTYPE_IPSTR: - case TAOS_CFG_VTYPE_DIRECTORY: - printf(" %s:%s%s%s\n", cfg->option, blank, (char *)cfg->ptr, tsGlobalUnit[cfg->unitType]); + case CFG_DTYPE_FLOAT: + if (dump) { + printf("%s %s %f", src, name, pItem->fval); + printf("\n"); + } else { + uInfo("%s %s %f", src, name, pItem->fval); + } break; - default: + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + if (dump) { + printf("%s %s %s", src, name, pItem->str); + printf("\n"); + } else { + uInfo("%s %s %s", src, name, pItem->str); + } break; } + } + + if (dump) { + printf("================================================================="); + printf("\n"); + } else { + uInfo("================================================================="); + } +} + +int32_t cfgLoadFromEnvVar(SConfig *pConfig) { + uInfo("load from global env variables"); + return 0; } -void taosDumpGlobalCfg() { - printf("taos global config:\n"); - printf("==================================\n"); - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW)) continue; +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { + uInfo("load from env file %s", filepath); + return 0; +} - taosDumpCfg(cfg); +int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { + char *line = NULL, *name, *value, *value2, *value3; + int32_t olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + + // FILE *fp = fopen(filepath, "r"); + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; } - printf("\ntaos local config:\n"); - printf("==================================\n"); + while (!taosEOFFile(pFile)) { + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = taosGetLineFile(pFile, &line); + if (_bytes < 0) { + break; + } + + line[_bytes - 1] = 0; + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; - for (int i = 0; i < tsGlobalConfigNum; ++i) { - SGlobalCfg *cfg = tsGlobalConfig + i; - if (tscEmbeddedInUtil == 0 && !(cfg->cfgType & TSDB_CFG_CTYPE_B_CLIENT)) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_NOT_PRINT) continue; - if (cfg->cfgType & TSDB_CFG_CTYPE_B_SHOW) continue; + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } - taosDumpCfg(cfg); + cfgSetItem(pConfig, name, value, CFG_STYPE_CFG_FILE); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_CFG_FILE); + } } + + taosCloseFile(&pFile); + if (line != NULL) tfree(line); + + uInfo("load from cfg file %s success", filepath); + return 0; } + +int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { + uInfo("load from apoll url %s", url); + return 0; +} \ No newline at end of file diff --git a/source/util/src/tcrc32c.c b/source/util/src/tcrc32c.c index 73bba4480e99f45a41ce937e9ea99bcd9077e995..f7e1060be0c42fa3704c9a471e5ae5719bdf90e2 100644 --- a/source/util/src/tcrc32c.c +++ b/source/util/src/tcrc32c.c @@ -18,7 +18,7 @@ 3. This notice may not be removed or altered from any source distribution. */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tcrc32c.h" #include "tdef.h" @@ -26,8 +26,6 @@ #include #endif - - #define POLY 0x82f63b78 #define LONG_SHIFT 8192 #define SHORT_SHIFT 256 @@ -1097,7 +1095,7 @@ static uint32_t short_shifts[4][256] = { static uint32_t append_trivial(uint32_t crc, crc_stream input, size_t length) { for (size_t i = 0; i < length; ++i) { crc = crc ^ input[i]; - for (int j = 0; j < 8; j++) + for (int32_t j = 0; j < 8; j++) crc = (crc >> 1) ^ 0x80000000 ^ ((~crc & 1) * POLY); } return crc; @@ -1358,7 +1356,7 @@ void taosResolveCRC() { #if defined _TD_ARM_ || defined _TD_MIPS_ || defined WINDOWS crc32c = crc32c_sf; #else - int sse42; + int32_t sse42; SSE42(sse42); crc32c = sse42 ? crc32c_hw : crc32c_sf; #endif @@ -1371,10 +1369,10 @@ void taosResolveCRC() { #include #include -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char str[1024] = "\0"; char *ptr = str; - int count = 0; + int32_t count = 0; while ((count = read(0, ptr, 10)) > 0) { ptr += count; } diff --git a/source/util/src/tdes.c b/source/util/src/tdes.c index f72ddcaa3bb1c63e06310f574dcaa765ed112046..105dd7f95fc204d11ad9233372984944760dfd70 100644 --- a/source/util/src/tdes.c +++ b/source/util/src/tdes.c @@ -13,43 +13,44 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tkey.h" +#define _DEFAULT_SOURCE +#include "tdes.h" #define ENCRYPTION_MODE 1 #define DECRYPTION_MODE 0 typedef struct { - unsigned char k[8]; - unsigned char c[4]; - unsigned char d[4]; + uint8_t k[8]; + uint8_t c[4]; + uint8_t d[4]; } key_set; -void generate_key(unsigned char* key); -void generate_sub_keys(unsigned char* main_key, key_set* key_sets); -void process_message(unsigned char* message_piece, unsigned char* processed_piece, key_set* key_sets, int mode); +void generate_key(uint8_t* key); +void generate_sub_keys(uint8_t* main_key, key_set* key_sets); +void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* key_sets, int32_t mode); #if 0 int64_t taosDesGenKey() { - unsigned int iseed = (unsigned int)time(NULL); + uint32_t iseed = (uint32_t)time(NULL); srand(iseed); - unsigned char key[8] = {0}; + uint8_t key[8] = {0}; generate_key(key); return *((int64_t*)key); } #endif -char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mode) { - unsigned int number_of_blocks = len / 8; - unsigned char data_block[9] = {0}; - unsigned char processed_block[9] = {0}; - key_set key_sets[17]; memset(key_sets, 0, sizeof(key_sets)); - char* dest = calloc(len + 1, 1); +char* taosDesImp(uint8_t* key, char* src, uint32_t len, int32_t process_mode) { + uint32_t number_of_blocks = len / 8; + uint8_t data_block[9] = {0}; + uint8_t processed_block[9] = {0}; + key_set key_sets[17]; + memset(key_sets, 0, sizeof(key_sets)); + char* dest = calloc(len + 1, 1); generate_sub_keys(key, key_sets); - for (unsigned int block_count = 0; block_count < number_of_blocks; block_count++) { + for (uint32_t block_count = 0; block_count < number_of_blocks; block_count++) { memset(processed_block, 0, 8); memcpy(data_block, src + block_count * 8, 8); process_message(data_block, processed_block, key_sets, process_mode); @@ -59,15 +60,15 @@ char* taosDesImp(unsigned char* key, char* src, unsigned int len, int process_mo return dest; } -char* taosDesEncode(int64_t key, char* src, int len) { +char* taosDesEncode(int64_t key, char* src, int32_t len) { if (len % 8 != 0) return NULL; - unsigned char* keyStr = (unsigned char*)(&key); + uint8_t* keyStr = (uint8_t*)(&key); return taosDesImp(keyStr, src, len, ENCRYPTION_MODE); } -char* taosDesDecode(int64_t key, char* src, int len) { - unsigned char* keyStr = (unsigned char*)(&key); - char* temp = calloc(len + 8, 1); +char* taosDesDecode(int64_t key, char* src, int32_t len) { + uint8_t* keyStr = (uint8_t*)(&key); + char* temp = calloc(len + 8, 1); memcpy(temp, src, len); len += 8; @@ -77,67 +78,67 @@ char* taosDesDecode(int64_t key, char* src, int len) { return decode; } -int initial_key_permutaion[] = {57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, - 35, 27, 19, 11, 3, 60, 52, 44, 36, 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, - 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4}; +int32_t initial_key_permutaion[] = {57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, + 35, 27, 19, 11, 3, 60, 52, 44, 36, 63, 55, 47, 39, 31, 23, 15, 7, 62, 54, + 46, 38, 30, 22, 14, 6, 61, 53, 45, 37, 29, 21, 13, 5, 28, 20, 12, 4}; -int initial_message_permutation[] = {58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, - 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, - 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, - 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7}; +int32_t initial_message_permutation[] = {58, 50, 42, 34, 26, 18, 10, 2, 60, 52, 44, 36, 28, 20, 12, 4, + 62, 54, 46, 38, 30, 22, 14, 6, 64, 56, 48, 40, 32, 24, 16, 8, + 57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, + 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7}; -int key_shift_sizes[] = {-1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1}; +int32_t key_shift_sizes[] = {-1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1}; -int sub_key_permutation[] = {14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, - 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47, 55, 30, 40, - 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32}; +int32_t sub_key_permutation[] = {14, 17, 11, 24, 1, 5, 3, 28, 15, 6, 21, 10, 23, 19, 12, 4, + 26, 8, 16, 7, 27, 20, 13, 2, 41, 52, 31, 37, 47, 55, 30, 40, + 51, 45, 33, 48, 44, 49, 39, 56, 34, 53, 46, 42, 50, 36, 29, 32}; -int message_expansion[] = {32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, - 12, 13, 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, - 22, 23, 24, 25, 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1}; +int32_t message_expansion[] = {32, 1, 2, 3, 4, 5, 4, 5, 6, 7, 8, 9, 8, 9, 10, 11, + 12, 13, 12, 13, 14, 15, 16, 17, 16, 17, 18, 19, 20, 21, 20, 21, + 22, 23, 24, 25, 24, 25, 26, 27, 28, 29, 28, 29, 30, 31, 32, 1}; -int S1[] = {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, 0, 15, 7, 4, 14, 2, - 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, - 3, 10, 5, 0, 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}; +int32_t S1[] = {14, 4, 13, 1, 2, 15, 11, 8, 3, 10, 6, 12, 5, 9, 0, 7, 0, 15, 7, 4, 14, 2, + 13, 1, 10, 6, 12, 11, 9, 5, 3, 8, 4, 1, 14, 8, 13, 6, 2, 11, 15, 12, 9, 7, + 3, 10, 5, 0, 15, 12, 8, 2, 4, 9, 1, 7, 5, 11, 3, 14, 10, 0, 6, 13}; -int S2[] = {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, 3, 13, 4, 7, 15, 2, - 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, - 9, 3, 2, 15, 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}; +int32_t S2[] = {15, 1, 8, 14, 6, 11, 3, 4, 9, 7, 2, 13, 12, 0, 5, 10, 3, 13, 4, 7, 15, 2, + 8, 14, 12, 0, 1, 10, 6, 9, 11, 5, 0, 14, 7, 11, 10, 4, 13, 1, 5, 8, 12, 6, + 9, 3, 2, 15, 13, 8, 10, 1, 3, 15, 4, 2, 11, 6, 7, 12, 0, 5, 14, 9}; -int S3[] = {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, 13, 7, 0, 9, 3, 4, - 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, - 5, 10, 14, 7, 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}; +int32_t S3[] = {10, 0, 9, 14, 6, 3, 15, 5, 1, 13, 12, 7, 11, 4, 2, 8, 13, 7, 0, 9, 3, 4, + 6, 10, 2, 8, 5, 14, 12, 11, 15, 1, 13, 6, 4, 9, 8, 15, 3, 0, 11, 1, 2, 12, + 5, 10, 14, 7, 1, 10, 13, 0, 6, 9, 8, 7, 4, 15, 14, 3, 11, 5, 2, 12}; -int S4[] = {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, 13, 8, 11, 5, 6, 15, - 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, - 5, 2, 8, 4, 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}; +int32_t S4[] = {7, 13, 14, 3, 0, 6, 9, 10, 1, 2, 8, 5, 11, 12, 4, 15, 13, 8, 11, 5, 6, 15, + 0, 3, 4, 7, 2, 12, 1, 10, 14, 9, 10, 6, 9, 0, 12, 11, 7, 13, 15, 1, 3, 14, + 5, 2, 8, 4, 3, 15, 0, 6, 10, 1, 13, 8, 9, 4, 5, 11, 12, 7, 2, 14}; -int S5[] = {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, 14, 11, 2, 12, 4, 7, - 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, - 6, 3, 0, 14, 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}; +int32_t S5[] = {2, 12, 4, 1, 7, 10, 11, 6, 8, 5, 3, 15, 13, 0, 14, 9, 14, 11, 2, 12, 4, 7, + 13, 1, 5, 0, 15, 10, 3, 9, 8, 6, 4, 2, 1, 11, 10, 13, 7, 8, 15, 9, 12, 5, + 6, 3, 0, 14, 11, 8, 12, 7, 1, 14, 2, 13, 6, 15, 0, 9, 10, 4, 5, 3}; -int S6[] = {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, 10, 15, 4, 2, 7, 12, - 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, - 1, 13, 11, 6, 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}; +int32_t S6[] = {12, 1, 10, 15, 9, 2, 6, 8, 0, 13, 3, 4, 14, 7, 5, 11, 10, 15, 4, 2, 7, 12, + 9, 5, 6, 1, 13, 14, 0, 11, 3, 8, 9, 14, 15, 5, 2, 8, 12, 3, 7, 0, 4, 10, + 1, 13, 11, 6, 4, 3, 2, 12, 9, 5, 15, 10, 11, 14, 1, 7, 6, 0, 8, 13}; -int S7[] = {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, 13, 0, 11, 7, 4, 9, - 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, - 0, 5, 9, 2, 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}; +int32_t S7[] = {4, 11, 2, 14, 15, 0, 8, 13, 3, 12, 9, 7, 5, 10, 6, 1, 13, 0, 11, 7, 4, 9, + 1, 10, 14, 3, 5, 12, 2, 15, 8, 6, 1, 4, 11, 13, 12, 3, 7, 14, 10, 15, 6, 8, + 0, 5, 9, 2, 6, 11, 13, 8, 1, 4, 10, 7, 9, 5, 0, 15, 14, 2, 3, 12}; -int S8[] = {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, 1, 15, 13, 8, 10, 3, - 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, - 15, 3, 5, 8, 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}; +int32_t S8[] = {13, 2, 8, 4, 6, 15, 11, 1, 10, 9, 3, 14, 5, 0, 12, 7, 1, 15, 13, 8, 10, 3, + 7, 4, 12, 5, 6, 11, 0, 14, 9, 2, 7, 11, 4, 1, 9, 12, 14, 2, 0, 6, 10, 13, + 15, 3, 5, 8, 2, 1, 14, 7, 4, 10, 8, 13, 15, 12, 9, 0, 3, 5, 6, 11}; -int right_sub_message_permutation[] = {16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, - 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25}; +int32_t right_sub_message_permutation[] = {16, 7, 20, 21, 29, 12, 28, 17, 1, 15, 23, 26, 5, 18, 31, 10, + 2, 8, 24, 14, 32, 27, 3, 9, 19, 13, 30, 6, 22, 11, 4, 25}; -int final_message_permutation[] = {40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, - 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, - 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, - 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25}; +int32_t final_message_permutation[] = {40, 8, 48, 16, 56, 24, 64, 32, 39, 7, 47, 15, 55, 23, 63, 31, + 38, 6, 46, 14, 54, 22, 62, 30, 37, 5, 45, 13, 53, 21, 61, 29, + 36, 4, 44, 12, 52, 20, 60, 28, 35, 3, 43, 11, 51, 19, 59, 27, + 34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25}; void print_char_as_binary(char input) { - int i; + int32_t i; for (i = 0; i < 8; i++) { char shift_byte = 0x01 << (7 - i); if (shift_byte & input) { @@ -148,15 +149,15 @@ void print_char_as_binary(char input) { } } -void generate_key(unsigned char* key) { - int i; +void generate_key(uint8_t* key) { + int32_t i; for (i = 0; i < 8; i++) { key[i] = taosRand() % 255; } } void print_key_set(key_set _key_set) { - int i; + int32_t i; printf("K: \n"); for (i = 0; i < 8; i++) { printf("%02X : ", _key_set.k[i]); @@ -180,10 +181,10 @@ void print_key_set(key_set _key_set) { printf("\n"); } -void generate_sub_keys(unsigned char* main_key, key_set* key_sets) { - int i, j; - int shift_size; - unsigned char shift_byte, first_shift_bits, second_shift_bits, third_shift_bits, fourth_shift_bits; +void generate_sub_keys(uint8_t* main_key, key_set* key_sets) { + int32_t i, j; + int32_t shift_size; + uint8_t shift_byte, first_shift_bits, second_shift_bits, third_shift_bits, fourth_shift_bits; for (i = 0; i < 8; i++) { key_sets[0].k[i] = 0; @@ -277,12 +278,12 @@ void generate_sub_keys(unsigned char* main_key, key_set* key_sets) { } } -void process_message(unsigned char* message_piece, unsigned char* processed_piece, key_set* key_sets, int mode) { - int i, k; - int shift_size; - unsigned char shift_byte; +void process_message(uint8_t* message_piece, uint8_t* processed_piece, key_set* key_sets, int32_t mode) { + int32_t i, k; + int32_t shift_size; + uint8_t shift_byte; - unsigned char initial_permutation[8]; + uint8_t initial_permutation[8]; memset(initial_permutation, 0, 8); memset(processed_piece, 0, 8); @@ -295,15 +296,15 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec initial_permutation[i / 8] |= (shift_byte >> i % 8); } - unsigned char l[4], r[4]; + uint8_t l[4], r[4]; for (i = 0; i < 4; i++) { l[i] = initial_permutation[i]; r[i] = initial_permutation[i + 4]; } - unsigned char ln[4], rn[4], er[6], ser[4]; + uint8_t ln[4], rn[4], er[6], ser[4]; - int key_index; + int32_t key_index; for (k = 1; k <= 16; k++) { memcpy(ln, r, 4); @@ -328,7 +329,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec er[i] ^= key_sets[key_index].k[i]; } - unsigned char row, column; + uint8_t row, column; for (i = 0; i < 4; i++) { ser[i] = 0; @@ -345,7 +346,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[0] & 0x78) >> 3); - ser[0] |= ((unsigned char)S1[row * 16 + column] << 4); + ser[0] |= ((uint8_t)S1[row * 16 + column] << 4); row = 0; row |= (er[0] & 0x02); @@ -355,7 +356,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[0] & 0x01) << 3); column |= ((er[1] & 0xE0) >> 5); - ser[0] |= (unsigned char)S2[row * 16 + column]; + ser[0] |= (uint8_t)S2[row * 16 + column]; // Byte 2 row = 0; @@ -366,7 +367,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[1] & 0x07) << 1); column |= ((er[2] & 0x80) >> 7); - ser[1] |= ((unsigned char)S3[row * 16 + column] << 4); + ser[1] |= ((uint8_t)S3[row * 16 + column] << 4); row = 0; row |= ((er[2] & 0x20) >> 4); @@ -375,7 +376,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[2] & 0x1E) >> 1); - ser[1] |= (unsigned char)S4[row * 16 + column]; + ser[1] |= (uint8_t)S4[row * 16 + column]; // Byte 3 row = 0; @@ -385,7 +386,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[3] & 0x78) >> 3); - ser[2] |= ((unsigned char)S5[row * 16 + column] << 4); + ser[2] |= ((uint8_t)S5[row * 16 + column] << 4); row = 0; row |= (er[3] & 0x02); @@ -395,7 +396,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[3] & 0x01) << 3); column |= ((er[4] & 0xE0) >> 5); - ser[2] |= (unsigned char)S6[row * 16 + column]; + ser[2] |= (uint8_t)S6[row * 16 + column]; // Byte 4 row = 0; @@ -406,7 +407,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column |= ((er[4] & 0x07) << 1); column |= ((er[5] & 0x80) >> 7); - ser[3] |= ((unsigned char)S7[row * 16 + column] << 4); + ser[3] |= ((uint8_t)S7[row * 16 + column] << 4); row = 0; row |= ((er[5] & 0x20) >> 4); @@ -415,7 +416,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec column = 0; column |= ((er[5] & 0x1E) >> 1); - ser[3] |= (unsigned char)S8[row * 16 + column]; + ser[3] |= (uint8_t)S8[row * 16 + column]; for (i = 0; i < 4; i++) { rn[i] = 0; @@ -440,7 +441,7 @@ void process_message(unsigned char* message_piece, unsigned char* processed_piec } } - unsigned char pre_end_permutation[8]; + uint8_t pre_end_permutation[8]; for (i = 0; i < 4; i++) { pre_end_permutation[i] = r[i]; pre_end_permutation[4 + i] = l[i]; diff --git a/source/util/src/encode.c b/source/util/src/tencode.c similarity index 96% rename from source/util/src/encode.c rename to source/util/src/tencode.c index 758d3f442db0007ea60317827d582d50f60f8d75..94b4cced46adb7f72e89b27b415a52ccf48a0253 100644 --- a/source/util/src/encode.c +++ b/source/util/src/tencode.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "encode.h" +#define _DEFAULT_SOURCE +#include "tencode.h" #if __STDC_VERSION__ >= 201112L static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); @@ -47,7 +48,7 @@ void tCoderClear(SCoder* pCoder) { } } -int tStartEncode(SCoder* pCoder) { +int32_t tStartEncode(SCoder* pCoder) { struct SCoderNode* pNode; ASSERT(pCoder->type == TD_ENCODER); @@ -96,7 +97,7 @@ void tEndEncode(SCoder* pCoder) { } } -int tStartDecode(SCoder* pCoder) { +int32_t tStartDecode(SCoder* pCoder) { int32_t len; struct SCoderNode* pNode; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2b53a769ff5e4ce1afbb14e68383c77fbbb10c73..c8cafb93dde0b08edc329ebba1394bb047c3518b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -13,32 +13,30 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" +#include "taoserror.h" #define TAOS_ERROR_C - + typedef struct { - int32_t val; - const char* str; + int32_t val; + const char* str; } STaosError; -#include "os.h" -#include "taoserror.h" - static threadlocal int32_t tsErrno; -int32_t* taosGetErrno() { - return &tsErrno; -} + +int32_t* taosGetErrno() { return &tsErrno; } #ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str=(msg)}, +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); #endif -#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) -#define TAOS_SUCCEEDED(err) ((err) >= 0) -#define TAOS_FAILED(err) ((err) < 0) +#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) +#define TAOS_SUCCEEDED(err) ((err) >= 0) +#define TAOS_FAILED(err) ((err) < 0) #ifdef TAOS_ERROR_C STaosError errors[] = { @@ -81,6 +79,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") +TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -243,6 +243,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_COLUMNS, "Too many columns") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") +// mnode-infoSchema +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name") + + // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") @@ -251,16 +255,16 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") // mnode-trans -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_INVALID_STAGE, "Invalid stage to kill") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CANT_PARALLEL, "Invalid stage to kill") // mnode-topic -TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") @@ -372,8 +376,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_MSG_ERROR, "Task message error") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_JOB_FREED, "Job already freed") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TASK_STATUS_ERROR, "Task status error") - - // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, "License expired") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DNODE_LIMITED, "DNode creation limited by licence") @@ -430,12 +432,11 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CTG_OUT_OF_SERVICE, "catalog is out of ser TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") - #ifdef TAOS_ERROR_C }; #endif -static int tsCompareTaosError(const void* a, const void* b) { +static int32_t taosCompareTaosError(const void* a, const void* b) { const STaosError* x = (const STaosError*)a; const STaosError* y = (const STaosError*)b; if (x->val < y->val) { @@ -448,11 +449,11 @@ static int tsCompareTaosError(const void* a, const void* b) { } static pthread_once_t tsErrorInit = PTHREAD_ONCE_INIT; + static void tsSortError(void) { - qsort(errors, sizeof(errors)/sizeof(errors[0]), sizeof(errors[0]), tsCompareTaosError); + qsort(errors, sizeof(errors) / sizeof(errors[0]), sizeof(errors[0]), taosCompareTaosError); } - const char* tstrerror(int32_t err) { pthread_once(&tsErrorInit, tsSortError); @@ -461,9 +462,11 @@ const char* tstrerror(int32_t err) { return strerror(err & 0x0000ffff); } - size_t s = 0, e = sizeof(errors)/sizeof(errors[0]); + int32_t s = 0; + int32_t e = sizeof(errors) / sizeof(errors[0]); + while (s < e) { - size_t mid = (s + e) / 2; + int32_t mid = (s + e) / 2; int32_t val = errors[mid].val; if (err > val) { s = mid + 1; diff --git a/source/util/src/texception.c b/source/util/src/texception.c new file mode 100644 index 0000000000000000000000000000000000000000..33befb694a0a0a71c01042d9c9b5b19f7da12912 --- /dev/null +++ b/source/util/src/texception.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "texception.h" + +static threadlocal SExceptionNode* expList; + +void exceptionPushNode(SExceptionNode* node) { + node->prev = expList; + expList = node; +} + +int32_t exceptionPopNode() { + SExceptionNode* node = expList; + expList = node->prev; + return node->code; +} + +void exceptionThrow(int32_t code) { + expList->code = code; + longjmp(expList->jb, 1); +} + +static void cleanupWrapper_void_ptr_ptr(SCleanupAction* ca) { + void (*func)(void*, void*) = ca->func; + func(ca->arg1.Ptr, ca->arg2.Ptr); +} + +static void cleanupWrapper_void_ptr_bool(SCleanupAction* ca) { + void (*func)(void*, bool) = ca->func; + func(ca->arg1.Ptr, ca->arg2.Bool); +} + +static void cleanupWrapper_void_ptr(SCleanupAction* ca) { + void (*func)(void*) = ca->func; + func(ca->arg1.Ptr); +} + +static void cleanupWrapper_int_int(SCleanupAction* ca) { + int32_t (*func)(int32_t) = ca->func; + func(ca->arg1.Int); +} + +static void cleanupWrapper_void(SCleanupAction* ca) { + void (*func)() = ca->func; + func(); +} + +static void cleanupWrapper_int_ptr(SCleanupAction* ca) { + int32_t (*func)(void*) = ca->func; + func(ca->arg1.Ptr); +} + +typedef void (*wrapper)(SCleanupAction*); +static wrapper wrappers[] = { + cleanupWrapper_void_ptr_ptr, cleanupWrapper_void_ptr_bool, cleanupWrapper_void_ptr, + cleanupWrapper_int_int, cleanupWrapper_void, cleanupWrapper_int_ptr, +}; + +void cleanupPush_void_ptr_ptr(bool failOnly, void* func, void* arg1, void* arg2) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 0; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Ptr = arg2; +} + +void cleanupPush_void_ptr_bool(bool failOnly, void* func, void* arg1, bool arg2) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 1; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg1; + ca->arg2.Bool = arg2; +} + +void cleanupPush_void_ptr(bool failOnly, void* func, void* arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 2; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} + +void cleanupPush_int_int(bool failOnly, void* func, int32_t arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 3; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Int = arg; +} + +void cleanupPush_void(bool failOnly, void* func) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 4; + ca->failOnly = failOnly; + ca->func = func; +} + +void cleanupPush_int_ptr(bool failOnly, void* func, void* arg) { + assert(expList->numCleanupAction < expList->maxCleanupAction); + + SCleanupAction* ca = expList->cleanupActions + expList->numCleanupAction++; + ca->wrapper = 5; + ca->failOnly = failOnly; + ca->func = func; + ca->arg1.Ptr = arg; +} + +int32_t cleanupGetActionCount() { return expList->numCleanupAction; } + +static void doExecuteCleanup(SExceptionNode* node, int32_t anchor, bool failed) { + while (node->numCleanupAction > anchor) { + --node->numCleanupAction; + SCleanupAction* ca = node->cleanupActions + node->numCleanupAction; + if (failed || !(ca->failOnly)) { + wrappers[ca->wrapper](ca); + } + } +} + +void cleanupExecuteTo(int32_t anchor, bool failed) { doExecuteCleanup(expList, anchor, failed); } + +void cleanupExecute(SExceptionNode* node, bool failed) { doExecuteCleanup(node, 0, failed); } +bool cleanupExceedLimit() { return expList->numCleanupAction >= expList->maxCleanupAction; } diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c deleted file mode 100644 index 0f68e9204d34557bf792e324dc69b73e9b759287..0000000000000000000000000000000000000000 --- a/source/util/src/tfile.c +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tref.h" -#include "tutil.h" -#include "ulog.h" - -static int32_t tsFileRsetId = -1; - -static int8_t tfInited = 0; - -static void tfCloseFile(void *p) { taosCloseFile((int32_t)(uintptr_t)p); } - -int32_t tfInit() { - int8_t old = atomic_val_compare_exchange_8(&tfInited, 0, 1); - if (old == 1) return 0; - tsFileRsetId = taosOpenRef(2000, tfCloseFile); - if (tsFileRsetId > 0) { - return 0; - } else { - atomic_store_8(&tfInited, 0); - return -1; - } -} - -void tfCleanup() { - atomic_store_8(&tfInited, 0); - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); - tsFileRsetId = -1; -} - -static int64_t tfOpenImp(int32_t fd) { - if (fd < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - void * p = (void *)(int64_t)fd; - int64_t rid = taosAddRef(tsFileRsetId, p); - if (rid < 0) taosCloseFile(fd); - - return rid; -} - -int64_t tfOpenRead(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileRead(pathname); - return tfOpenImp(fd); -} - -int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { - int32_t fd = taosOpenFileReadWrite(pathname); - return tfOpenImp(fd); -} - -int64_t tfOpenCreateWrite(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWrite(pathname); - return tfOpenImp(fd); -} - -int64_t tfOpenCreateWriteAppend(const char *pathname, int32_t flags, mode_t mode) { - int32_t fd = taosOpenFileCreateWriteAppend(pathname); - return tfOpenImp(fd); -} - -int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } - -int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - - int64_t ret = taosWriteFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} - -int64_t tfRead(int64_t tfd, void *buf, int64_t count) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - - int64_t ret = taosReadFile(fd, buf, count); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} - -int64_t tfPread(int64_t tfd, void *buf, int64_t count, int32_t offset) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - - int64_t ret = pread(fd, buf, count, offset); - if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); - - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} - -int32_t tfFsync(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFsyncFile(fd); - - taosReleaseRef(tsFileRsetId, tfd); - return code; -} - -bool tfValid(int64_t tfd) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return false; - - taosReleaseRef(tsFileRsetId, tfd); - return true; -} - -int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - int64_t ret = taosLSeekFile(fd, offset, whence); - - taosReleaseRef(tsFileRsetId, tfd); - return ret; -} - -int32_t tfFtruncate(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return -1; - - int32_t fd = (int32_t)(uintptr_t)p; - int32_t code = taosFtruncateFile(fd, length); - - taosReleaseRef(tsFileRsetId, tfd); - return code; -} - -void *tfMmapReadOnly(int64_t tfd, int64_t length) { - void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return NULL; - int32_t fd = (int32_t)(uintptr_t)p; - - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, fd, 0); - taosReleaseRef(tsFileRsetId, tfd); - return ptr; -} diff --git a/source/util/src/tfunctional.c b/source/util/src/tfunctional.c index 8b20f8fc0a6b6e08bcfd501f625c4594b4dff50d..c49fbdb50438effa0d980d50aaa00ea4a2d9428b 100644 --- a/source/util/src/tfunctional.c +++ b/source/util/src/tfunctional.c @@ -13,37 +13,34 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tfunctional.h" -tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int numOfArgs) { +tGenericSavedFunc* genericSavedFuncInit(GenericVaFunc func, int32_t numOfArgs) { tGenericSavedFunc* pSavedFunc = malloc(sizeof(tGenericSavedFunc) + numOfArgs * (sizeof(void*))); - if(pSavedFunc == NULL) return NULL; + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int numOfArgs) { - tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void *)); - if(pSavedFunc == NULL) return NULL; +tI32SavedFunc* i32SavedFuncInit(I32VaFunc func, int32_t numOfArgs) { + tI32SavedFunc* pSavedFunc = malloc(sizeof(tI32SavedFunc) + numOfArgs * sizeof(void*)); + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int numOfArgs) { +tVoidSavedFunc* voidSavedFuncInit(VoidVaFunc func, int32_t numOfArgs) { tVoidSavedFunc* pSavedFunc = malloc(sizeof(tVoidSavedFunc) + numOfArgs * sizeof(void*)); - if(pSavedFunc == NULL) return NULL; + if (pSavedFunc == NULL) return NULL; pSavedFunc->func = func; return pSavedFunc; } -FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { - return pSavedFunc->func(pSavedFunc->args); -} +FORCE_INLINE void* genericInvoke(tGenericSavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); } -FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { - return pSavedFunc->func(pSavedFunc->args); -} +FORCE_INLINE int32_t i32Invoke(tI32SavedFunc* const pSavedFunc) { return pSavedFunc->func(pSavedFunc->args); } FORCE_INLINE void voidInvoke(tVoidSavedFunc* const pSavedFunc) { - if(pSavedFunc) pSavedFunc->func(pSavedFunc->args); + if (pSavedFunc) pSavedFunc->func(pSavedFunc->args); } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 0b0d8c4c589ee9f5effa33e17f19573ab0a51199..219fc739ca845e1abf5b6a55e1ce1c0a87cb6149 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -13,16 +13,15 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "thash.h" -#include "ulog.h" -#include "taos.h" #include "tdef.h" +#include "tlog.h" // the add ref count operation may trigger the warning if the reference count is greater than the MAX_WARNING_REF_COUNT #define MAX_WARNING_REF_COUNT 10000 #define EXT_SIZE 1024 -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define DO_FREE_HASH_NODE(_n) \ do { \ @@ -74,10 +73,12 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntry *pe, const void *key, size_t keyLen, + uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -115,7 +116,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, + SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); pNode->count--; @@ -129,11 +131,11 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pNewNode->next = pNode->next; DO_FREE_HASH_NODE(pNode); } else { - pNewNode->next = pNode; + pNewNode->next = pNode; pe->num++; atomic_add_fetch_32(&pHashObj->size, 1); } - + return pNewNode; } @@ -175,7 +177,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->capacity = taosHashCapacity((int32_t)capacity); assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->equalFp = memcmp; - pHashObj->hashFp = fn; + pHashObj->hashFp = fn; pHashObj->type = type; pHashObj->enableUpdate = update; @@ -201,8 +203,8 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp) { if (pHashObj != NULL && fp != NULL) { pHashObj->equalFp = fp; - } -} + } +} int32_t taosHashGetSize(const SHashObj *pHashObj) { if (!pHashObj) { @@ -211,9 +213,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { return (int32_t)atomic_load_32(&pHashObj->size); } -static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { - return taosHashGetSize(pHashObj) == 0; -} +static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return taosHashGetSize(pHashObj) == 0; } int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded) { uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -224,12 +224,12 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock((void*) &pHashObj->lock, pHashObj->type); + __wr_lock((void *)&pHashObj->lock, pHashObj->type); taosHashTableResize(pHashObj); - __wr_unlock((void*) &pHashObj->lock, pHashObj->type); + __wr_unlock((void *)&pHashObj->lock, pHashObj->type); } - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -245,9 +245,10 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void assert(pNode == NULL); } - SHashNode* prev = NULL; + SHashNode *prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -271,13 +272,13 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void } // enable resize - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); atomic_add_fetch_32(&pHashObj->size, 1); if (newAdded) { *newAdded = true; } - + return 0; } else { // not support the update operation, return error @@ -292,7 +293,7 @@ int32_t taosHashPutImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void } // enable resize - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); if (newAdded) { *newAdded = false; @@ -310,13 +311,13 @@ int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void return taosHashPutImpl(pHashObj, key, keyLen, data, size, newAdded); } - void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetClone(pHashObj, key, keyLen, NULL); } -//TODO(yihaoDeng), merge with taosHashGetClone -void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void** d, size_t *sz) { +// TODO(yihaoDeng), merge with taosHashGetClone +void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, + size_t *sz) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -324,14 +325,14 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return NULL; } @@ -353,19 +354,19 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo if (fp != NULL) { fp(GET_HASH_NODE_DATA(pNode)); } - + if (*d == NULL) { - *sz = pNode->dataLen + EXT_SIZE; - *d = calloc(1, *sz); - } else if (*sz < pNode->dataLen){ *sz = pNode->dataLen + EXT_SIZE; - *d = realloc(*d, *sz); + *d = calloc(1, *sz); + } else if (*sz < pNode->dataLen) { + *sz = pNode->dataLen + EXT_SIZE; + *d = realloc(*d, *sz); } memcpy((char *)(*d), GET_HASH_NODE_DATA(pNode), pNode->dataLen); - // just make runtime happy + // just make runtime happy if ((*sz) - pNode->dataLen > 0) { memset((char *)(*d) + pNode->dataLen, 0, (*sz) - pNode->dataLen); - } + } data = GET_HASH_NODE_DATA(pNode); } @@ -374,11 +375,11 @@ void* taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, vo taosRUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void* d, bool acquire) { +void *taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void *d, bool acquire) { if (taosHashTableEmpty(pHashObj) || keyLen == 0 || key == NULL) { return NULL; } @@ -386,14 +387,14 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (atomic_load_32(&pe->num) == 0) { - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return NULL; } @@ -431,20 +432,19 @@ void* taosHashGetCloneImpl(SHashObj *pHashObj, const void *key, size_t keyLen, v taosRUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } -void* taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* d) { +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *d) { return taosHashGetCloneImpl(pHashObj, key, keyLen, d, false); } -void* taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { +void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashGetCloneImpl(pHashObj, key, keyLen, NULL, true); } - -int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, void *data, size_t dsize*/) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen /*, void *data, size_t dsize*/) { if (pHashObj == NULL || taosHashTableEmpty(pHashObj)) { return -1; } @@ -452,7 +452,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); // disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -466,23 +466,23 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi assert(pe->next == NULL); taosWUnLockLatch(&pe->latch); - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return -1; } - int code = -1; + int32_t code = -1; SHashNode *pNode = pe->next; SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) + if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && + pNode->removed == 0) break; prevNode = pNode; pNode = pNode->next; } - if (pNode) { code = 0; // it is found @@ -495,7 +495,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi pe->next = pNode->next; } -// if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); + // if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); pe->num--; atomic_sub_fetch_32(&pHashObj->size, 1); @@ -507,7 +507,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen/*, voi taosWUnLockLatch(&pe->latch); } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return code; } @@ -518,7 +518,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi } // disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); int32_t numOfEntries = (int32_t)pHashObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { @@ -533,7 +533,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // todo remove the first node SHashNode *pNode = NULL; - while((pNode = pEntry->next) != NULL) { + while ((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; atomic_sub_fetch_32(&pHashObj->size, 1); @@ -582,7 +582,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi } } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return 0; } @@ -593,7 +593,7 @@ void taosHashClear(SHashObj *pHashObj) { SHashNode *pNode, *pNext; - __wr_lock((void*) &pHashObj->lock, pHashObj->type); + __wr_lock((void *)&pHashObj->lock, pHashObj->type); for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; @@ -617,7 +617,7 @@ void taosHashClear(SHashObj *pHashObj) { } atomic_store_32(&pHashObj->size, 0); - __wr_unlock((void*) &pHashObj->lock, pHashObj->type); + __wr_unlock((void *)&pHashObj->lock, pHashObj->type); } void taosHashCleanup(SHashObj *pHashObj) { @@ -676,7 +676,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; @@ -685,7 +685,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newSize - pHashObj->capacity; - void * p = calloc(inc, sizeof(SHashEntry)); + void *p = calloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -756,15 +756,13 @@ void taosHashTableResize(SHashObj *pHashObj) { } else { assert(pe->next != NULL); } - } - } int64_t et = taosGetTimestampUs(); uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity, - ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -775,12 +773,12 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return NULL; } - pNewNode->keyLen = (uint32_t)keyLen; + pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; - pNewNode->dataLen = (uint32_t) dsize; - pNewNode->count = 1; + pNewNode->dataLen = (uint32_t)dsize; + pNewNode->count = 1; pNewNode->removed = 0; - pNewNode->next = NULL; + pNewNode->next = NULL; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -802,15 +800,16 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + + sizeof(SHashObj); } -FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { +FORCE_INLINE int32_t taosHashGetKey(void *data, void **key, size_t *keyLen) { if (NULL == data || NULL == key) { return -1; } - - SHashNode * node = GET_HASH_PNODE(data); + + SHashNode *node = GET_HASH_PNODE(data); *key = GET_HASH_NODE_KEY(node); if (keyLen) { *keyLen = node->keyLen; @@ -820,19 +819,17 @@ FORCE_INLINE int32_t taosHashGetKey(void *data, void** key, size_t* keyLen) { } FORCE_INLINE int32_t taosHashGetDataLen(void *data) { - SHashNode * node = GET_HASH_PNODE(data); + SHashNode *node = GET_HASH_PNODE(data); return node->keyLen; } FORCE_INLINE uint32_t taosHashGetDataKeyLen(SHashObj *pHashObj, void *data) { - SHashNode * node = GET_HASH_PNODE(data); + SHashNode *node = GET_HASH_PNODE(data); return node->keyLen; } - // release the pNode, return next pNode, and lock the current entry -static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { - +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int32_t *slot) { SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); SHashNode *prevNode = NULL; @@ -847,14 +844,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { SHashNode *pNode = pe->next; while (pNode) { - if (pNode == pOld) - break; + if (pNode == pOld) break; prevNode = pNode; pNode = pNode->next; } - if (pNode) { + if (pNode) { pNode = pNode->next; while (pNode) { if (pNode->removed == 0) break; @@ -862,17 +858,17 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } pOld->count--; - if (pOld->count <=0) { + if (pOld->count <= 0) { if (prevNode) { prevNode->next = pOld->next; } else { pe->next = pOld->next; } - + pe->num--; atomic_sub_fetch_32(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pOld); - } + } } else { uError("pNode:%p data:%p is not there!!!", pNode, p); } @@ -881,13 +877,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { } void *taosHashIterate(SHashObj *pHashObj, void *p) { - if (pHashObj == NULL) return NULL; + if (pHashObj == NULL) return NULL; - int slot = 0; + int32_t slot = 0; char *data = NULL; // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); SHashNode *pNode = NULL; if (p) { @@ -896,7 +892,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { SHashEntry *pe = pHashObj->hashList[slot]; if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } slot = slot + 1; } @@ -921,7 +917,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } } } @@ -947,10 +943,10 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } + } } - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); return data; } @@ -958,21 +954,17 @@ void taosHashCancelIterate(SHashObj *pHashObj, void *p) { if (pHashObj == NULL || p == NULL) return; // only add the read lock to disable the resize process - __rd_lock((void*) &pHashObj->lock, pHashObj->type); + __rd_lock((void *)&pHashObj->lock, pHashObj->type); - int slot; + int32_t slot; taosHashReleaseNode(pHashObj, p, &slot); SHashEntry *pe = pHashObj->hashList[slot]; if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); - } - - __rd_unlock((void*) &pHashObj->lock, pHashObj->type); -} + } -void taosHashRelease(SHashObj *pHashObj, void *p) { - taosHashCancelIterate(pHashObj, p); + __rd_unlock((void *)&pHashObj->lock, pHashObj->type); } - +void taosHashRelease(SHashObj *pHashObj, void *p) { taosHashCancelIterate(pHashObj, p); } diff --git a/source/util/src/thashutil.c b/source/util/src/thashutil.c index 2d7f8a5b032db7a2bfbc1d2cc6f6b541f76b1977..d5182cb892d2d314de52db7594768c00ad2808f1 100644 --- a/source/util/src/thashutil.c +++ b/source/util/src/thashutil.c @@ -13,10 +13,9 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "thash.h" -#include "compare.h" -#include "taos.h" +#include "tcompare.h" #include "types.h" #define ROTL32(x, r) ((x) << (r) | (x) >> (32u - (r))) @@ -34,7 +33,7 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint8_t *data = (const uint8_t *)key; - const int nblocks = len >> 2u; + const int32_t nblocks = len >> 2u; uint32_t h1 = 0x12345678; @@ -43,7 +42,7 @@ uint32_t MurmurHash3_32(const char *key, uint32_t len) { const uint32_t *blocks = (const uint32_t *)(data + nblocks * 4); - for (int i = -nblocks; i; i++) { + for (int32_t i = -nblocks; i; i++) { uint32_t k1 = blocks[i]; k1 *= c1; @@ -92,7 +91,7 @@ uint32_t taosFloatHash(const char *key, uint32_t UNUSED_PARAM(len)) { return 0; } if (fabs(f) < FLT_MAX/BASE - DLT) { - int t = (int)(round(BASE * (f + DLT))); + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; @@ -108,7 +107,7 @@ uint32_t taosDoubleHash(const char *key, uint32_t UNUSED_PARAM(len)) { return 0; } if (fabs(f) < DBL_MAX/BASE - DLT) { - int t = (int)(round(BASE * (f + DLT))); + int32_t t = (int32_t)(round(BASE * (f + DLT))); return (uint32_t)t; } else { return 0x7fc00000; diff --git a/source/util/src/theap.c b/source/util/src/theap.c index aa822c7d5ec1d5f107385a53d77cc104c30b14a4..30af0483cc86c91b5ba2ba7a6fff81649f064a65 100644 --- a/source/util/src/theap.c +++ b/source/util/src/theap.c @@ -13,15 +13,16 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "theap.h" -size_t heapSize(Heap* heap) { - return heap->nelts; -} +size_t heapSize(Heap* heap) { return heap->nelts; } Heap* heapCreate(HeapCompareFn fn) { - Heap *heap = calloc(1, sizeof(Heap)); - if (heap == NULL) { return NULL; } + Heap* heap = calloc(1, sizeof(Heap)); + if (heap == NULL) { + return NULL; + } heap->min = NULL; heap->nelts = 0; @@ -29,18 +30,14 @@ Heap* heapCreate(HeapCompareFn fn) { return heap; } -void heapDestroy(Heap *heap) { - free(heap); -} +void heapDestroy(Heap* heap) { free(heap); } -HeapNode* heapMin(const Heap* heap) { - return heap->min; -} +HeapNode* heapMin(const Heap* heap) { return heap->min; } /* Swap parent with child. Child moves closer to the root, parent moves away. */ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { HeapNode* sibling; - HeapNode t; + HeapNode t; t = *parent; *parent = *child; @@ -54,13 +51,10 @@ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { child->right = parent; sibling = child->left; } - if (sibling != NULL) - sibling->parent = child; + if (sibling != NULL) sibling->parent = child; - if (parent->left != NULL) - parent->left->parent = parent; - if (parent->right != NULL) - parent->right->parent = parent; + if (parent->left != NULL) parent->left->parent = parent; + if (parent->right != NULL) parent->right->parent = parent; if (child->parent == NULL) heap->min = child; @@ -73,9 +67,9 @@ static void heapNodeSwap(Heap* heap, HeapNode* parent, HeapNode* child) { void heapInsert(Heap* heap, HeapNode* newnode) { HeapNode** parent; HeapNode** child; - unsigned int path; - unsigned int n; - unsigned int k; + uint32_t path; + uint32_t n; + uint32_t k; newnode->left = NULL; newnode->right = NULL; @@ -85,8 +79,7 @@ void heapInsert(Heap* heap, HeapNode* newnode) { * heap so we always insert at the left-most free node of the bottom row. */ path = 0; - for (k = 0, n = 1 + heap->nelts; n >= 2; k += 1, n /= 2) - path = (path << 1) | (n & 1); + for (k = 0, n = 1 + heap->nelts; n >= 2; k += 1, n /= 2) path = (path << 1) | (n & 1); /* Now traverse the heap using the path we calculated in the previous step. */ parent = child = &heap->min; @@ -113,22 +106,20 @@ void heapInsert(Heap* heap, HeapNode* newnode) { } void heapRemove(Heap* heap, HeapNode* node) { - HeapNode* smallest; + HeapNode* smallest; HeapNode** max; - HeapNode* child; - unsigned int path; - unsigned int k; - unsigned int n; + HeapNode* child; + uint32_t path; + uint32_t k; + uint32_t n; - if (heap->nelts == 0) - return; + if (heap->nelts == 0) return; /* Calculate the path from the min (the root) to the max, the left-most node * of the bottom row. */ path = 0; - for (k = 0, n = heap->nelts; n >= 2; k += 1, n /= 2) - path = (path << 1) | (n & 1); + for (k = 0, n = heap->nelts; n >= 2; k += 1, n /= 2) path = (path << 1) | (n & 1); /* Now traverse the heap using the path we calculated in the previous step. */ max = &heap->min; @@ -182,12 +173,9 @@ void heapRemove(Heap* heap, HeapNode* node) { */ for (;;) { smallest = child; - if (child->left != NULL && (heap->compFn)(child->left, smallest)) - smallest = child->left; - if (child->right != NULL && (heap->compFn)(child->right, smallest)) - smallest = child->right; - if (smallest == child) - break; + if (child->left != NULL && (heap->compFn)(child->left, smallest)) smallest = child->left; + if (child->right != NULL && (heap->compFn)(child->right, smallest)) smallest = child->right; + if (smallest == child) break; heapNodeSwap(heap, child, smallest); } @@ -195,12 +183,7 @@ void heapRemove(Heap* heap, HeapNode* node) { * this is required, because `max` node is not guaranteed to be the * actual maximum in tree */ - while (child->parent != NULL && (heap->compFn)(child, child->parent)) - heapNodeSwap(heap, child->parent, child); -} - -void heapDequeue(Heap* heap) { - heapRemove(heap, heap->min); + while (child->parent != NULL && (heap->compFn)(child, child->parent)) heapNodeSwap(heap, child->parent, child); } - +void heapDequeue(Heap* heap) { heapRemove(heap, heap->min); } diff --git a/source/util/src/tidpool.c b/source/util/src/tidpool.c index 00c43bb25fdf6cea9d618b0ad5c48a0951a93b35..3ae537eae8a9b7c5b16bc91af5c994c140179ee4 100644 --- a/source/util/src/tidpool.c +++ b/source/util/src/tidpool.c @@ -13,18 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" -#include "ulog.h" - -typedef struct { - int maxId; - int numOfFree; - int freeSlot; - bool * freeList; - pthread_mutex_t mutex; -} id_pool_t; - -void *taosInitIdPool(int maxId) { +#define _DEFAULT_SOURCE +#include "tidpool.h" +#include "tlog.h" + +void *taosInitIdPool(int32_t maxId) { id_pool_t *pIdPool = calloc(1, sizeof(id_pool_t)); if (pIdPool == NULL) return NULL; @@ -45,17 +38,14 @@ void *taosInitIdPool(int maxId) { return pIdPool; } -int taosAllocateId(void *handle) { - id_pool_t *pIdPool = handle; - if (handle == NULL) { - return -1; - } +int32_t taosAllocateId(id_pool_t *pIdPool) { + if (pIdPool == NULL) return -1; - int slot = -1; + int32_t slot = -1; pthread_mutex_lock(&pIdPool->mutex); if (pIdPool->numOfFree > 0) { - for (int i = 0; i < pIdPool->maxId; ++i) { + for (int32_t i = 0; i < pIdPool->maxId; ++i) { slot = (i + pIdPool->freeSlot) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { pIdPool->freeList[slot] = true; @@ -70,13 +60,12 @@ int taosAllocateId(void *handle) { return slot + 1; } -void taosFreeId(void *handle, int id) { - id_pool_t *pIdPool = handle; - if (handle == NULL) return; +void taosFreeId(id_pool_t *pIdPool, int32_t id) { + if (pIdPool == NULL) return; pthread_mutex_lock(&pIdPool->mutex); - int slot = (id - 1) % pIdPool->maxId; + int32_t slot = (id - 1) % pIdPool->maxId; if (pIdPool->freeList[slot]) { pIdPool->freeList[slot] = false; pIdPool->numOfFree++; @@ -85,9 +74,7 @@ void taosFreeId(void *handle, int id) { pthread_mutex_unlock(&pIdPool->mutex); } -void taosIdPoolCleanUp(void *handle) { - id_pool_t *pIdPool = handle; - +void taosIdPoolCleanUp(id_pool_t *pIdPool) { if (pIdPool == NULL) return; uDebug("pool:%p is cleaned", pIdPool); @@ -101,22 +88,19 @@ void taosIdPoolCleanUp(void *handle) { free(pIdPool); } -int taosIdPoolNumOfUsed(void *handle) { - id_pool_t *pIdPool = handle; - +int32_t taosIdPoolNumOfUsed(id_pool_t *pIdPool) { pthread_mutex_lock(&pIdPool->mutex); - int ret = pIdPool->maxId - pIdPool->numOfFree; + int32_t ret = pIdPool->maxId - pIdPool->numOfFree; pthread_mutex_unlock(&pIdPool->mutex); return ret; } -bool taosIdPoolMarkStatus(void *handle, int id) { +bool taosIdPoolMarkStatus(id_pool_t *pIdPool, int32_t id) { bool ret = false; - id_pool_t *pIdPool = handle; pthread_mutex_lock(&pIdPool->mutex); - int slot = (id - 1) % pIdPool->maxId; + int32_t slot = (id - 1) % pIdPool->maxId; if (!pIdPool->freeList[slot]) { pIdPool->freeList[slot] = true; pIdPool->numOfFree--; @@ -129,8 +113,7 @@ bool taosIdPoolMarkStatus(void *handle, int id) { return ret; } -int taosUpdateIdPool(id_pool_t *handle, int maxId) { - id_pool_t *pIdPool = (id_pool_t*)handle; +int32_t taosUpdateIdPool(id_pool_t *pIdPool, int32_t maxId) { if (maxId <= pIdPool->maxId) { return 0; } @@ -155,11 +138,9 @@ int taosUpdateIdPool(id_pool_t *handle, int maxId) { return 0; } -int taosIdPoolMaxSize(void *handle) { - id_pool_t *pIdPool = (id_pool_t *)handle; - +int32_t taosIdPoolMaxSize(id_pool_t *pIdPool) { pthread_mutex_lock(&pIdPool->mutex); - int ret = pIdPool->maxId; + int32_t ret = pIdPool->maxId; pthread_mutex_unlock(&pIdPool->mutex); return ret; diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 13367843fc6e5400b4aba3090b2939d783cca812..4b68467450a1e2f8b1313ca81953e295eb752483 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -13,22 +13,18 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tjson.h" - -#include "taoserror.h" #include "cJSON.h" +#include "taoserror.h" -SJson* tjsonCreateObject() { - return cJSON_CreateObject(); -} +SJson* tjsonCreateObject() { return cJSON_CreateObject(); } -void tjsonDelete(SJson* pJson) { - cJSON_Delete((cJSON*)pJson); -} +void tjsonDelete(SJson* pJson) { cJSON_Delete((cJSON*)pJson); } int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number) { char tmp[40] = {0}; - snprintf(tmp, tListLen(tmp), "%"PRId64, number); + snprintf(tmp, tListLen(tmp), "%" PRId64, number); return tjsonAddStringToObject(pJson, pName, tmp); } @@ -36,15 +32,17 @@ int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double num return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } +int32_t tjsonAddBoolToObject(SJson* pJson, const char* pName, const bool boolean) { + return (NULL == cJSON_AddBoolToObject((cJSON*)pJson, pName, boolean) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); +} + int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); } -SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { - return cJSON_AddArrayToObject((cJSON*)pJson, pName); -} +SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName) { return cJSON_AddArrayToObject((cJSON*)pJson, pName); } -int32_t tjsonAddItemToObject(SJson *pJson, const char* pName, SJson* pItem) { +int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem) { return (cJSON_AddItemToObject((cJSON*)pJson, pName, pItem) ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); } @@ -75,10 +73,106 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) { return tjsonAddItemToArray(pJson, pJobj); } -char* tjsonToString(const SJson* pJson) { - return cJSON_Print((cJSON*)pJson); +char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } + +char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } + +SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); } + +int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { + char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); + if (NULL == p) { + return TSDB_CODE_FAILED; + } + strcpy(pVal, p); + return TSDB_CODE_SUCCESS; +} + +int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal) { + char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); + if (NULL == p) { + return TSDB_CODE_FAILED; + } + *pVal = strdup(p); + return TSDB_CODE_SUCCESS; +} + +int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal) { + char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); + if (NULL == p) { + return TSDB_CODE_FAILED; + } + char* pEnd = NULL; + *pVal = strtol(p, &pEnd, 10); + return (NULL == pEnd ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); +} + +int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal) { + int64_t val = 0; + int32_t code = tjsonGetBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + +int32_t tjsonGetSmallIntValue(const SJson* pJson, const char* pName, int16_t* pVal) { + int64_t val = 0; + int32_t code = tjsonGetBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + +int32_t tjsonGetTinyIntValue(const SJson* pJson, const char* pName, int8_t* pVal) { + int64_t val = 0; + int32_t code = tjsonGetBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + +int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pVal) { + char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); + if (NULL == p) { + return TSDB_CODE_FAILED; + } + char* pEnd = NULL; + *pVal = strtoul(p, &pEnd, 10); + return (NULL == pEnd ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED); +} + +int32_t tjsonGetUTinyIntValue(const SJson* pJson, const char* pName, uint8_t* pVal) { + uint64_t val = 0; + int32_t code = tjsonGetUBigIntValue(pJson, pName, &val); + *pVal = val; + return code; +} + +int32_t tjsonGetBoolValue(const SJson* pJson, const char* pName, bool* pVal) { + const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (cJSON_IsBool(pObject)) { + return TSDB_CODE_FAILED; + } + *pVal = cJSON_IsTrue(pObject) ? true : false; + return TSDB_CODE_SUCCESS; +} + +int32_t tjsonGetDoubleValue(const SJson* pJson, const char* pName, double* pVal) { + const SJson* pObject = tjsonGetObjectItem(pJson, pName); + if (!cJSON_IsNumber(pObject)) { + return TSDB_CODE_FAILED; + } + *pVal = cJSON_GetNumberValue(pObject); + return TSDB_CODE_SUCCESS; +} + +int32_t tjsonGetArraySize(const SJson* pJson) { return cJSON_GetArraySize(pJson); } + +SJson* tjsonGetArrayItem(const SJson* pJson, int32_t index) { return cJSON_GetArrayItem(pJson, index); } + +int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj) { + SJson* pJsonObj = tjsonGetObjectItem(pJson, pName); + if (NULL == pJsonObj) { + return TSDB_CODE_FAILED; + } + return func(pJsonObj, pObj); } -char* tjsonToUnformattedString(const SJson* pJson) { - return cJSON_PrintUnformatted((cJSON*)pJson); -} \ No newline at end of file +SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index f79bca1e4b03377cdfdaeda9ad4adf06ddff6de4..5fccba614b5c65beb6893ad0be64a61732bb200d 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -13,15 +13,15 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tlist.h" -#include "os.h" -void tdListInit(SList *list, int eleSize) { +void tdListInit(SList *list, int32_t eleSize) { TD_DLIST_INIT(list); listEleSize(list) = eleSize; } -SList *tdListNew(int eleSize) { +SList *tdListNew(int32_t eleSize) { SList *list = (SList *)malloc(sizeof(SList)); if (list == NULL) return NULL; @@ -50,7 +50,7 @@ void tdListPrependNode(SList *list, SListNode *node) { TD_DLIST_PREPEND(list, no void tdListAppendNode(SList *list, SListNode *node) { TD_DLIST_APPEND(list, node); } -int tdListPrepend(SList *list, void *data) { +int32_t tdListPrepend(SList *list, void *data) { SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; @@ -60,7 +60,7 @@ int tdListPrepend(SList *list, void *data) { return 0; } -int tdListAppend(SList *list, void *data) { +int32_t tdListAppend(SList *list, void *data) { SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; diff --git a/source/util/src/tlockfree.c b/source/util/src/tlockfree.c index f54206f5cbeee046fb009f9f3068b6305cd535e9..8edcca0c1635498e4fc25fa54028c26435f664f9 100644 --- a/source/util/src/tlockfree.c +++ b/source/util/src/tlockfree.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tlockfree.h" #define TD_RWLATCH_WRITE_FLAG 0x40000000 @@ -22,7 +22,7 @@ void taosInitRWLatch(SRWLatch *pLatch) { *pLatch = 0; } void taosWLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; - int nLoops = 0; + int32_t nLoops = 0; // Set write flag while (1) { @@ -57,7 +57,7 @@ void taosWUnLockLatch(SRWLatch *pLatch) { atomic_store_32(pLatch, 0); } void taosRLockLatch(SRWLatch *pLatch) { SRWLatch oLatch, nLatch; - int nLoops = 0; + int32_t nLoops = 0; while (1) { oLatch = atomic_load_32(pLatch); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 12aa77214f6109e9a33082c428a7bb7722a62835..d821e0440b814f28e6e9274261f21fffac10b00d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -15,32 +15,29 @@ #define _DEFAULT_SOURCE #include "tlog.h" -#include "os.h" -#include "tnote.h" #include "tutil.h" -#include "ulog.h" -#define MAX_LOGLINE_SIZE (1000) -#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) -#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) -#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) -#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) -#define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) +#define LOG_MAX_LINE_SIZE (1000) +#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 10) +#define LOG_MAX_LINE_CONTENT_SIZE (LOG_MAX_LINE_SIZE - 100) +#define LOG_MAX_LINE_DUMP_SIZE (65 * 1024) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 10) +#define LOG_MAX_LINE_DUMP_CONTENT_SIZE (LOG_MAX_LINE_DUMP_SIZE - 100) -#define LOG_FILE_NAME_LEN 300 -#define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB +#define LOG_FILE_NAME_LEN 300 +#define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB -#define DEFAULT_LOG_INTERVAL 25 -#define LOG_INTERVAL_STEP 5 -#define MIN_LOG_INTERVAL 5 -#define MAX_LOG_INTERVAL 25 -#define LOG_MAX_WAIT_MSEC 1000 +#define LOG_DEFAULT_INTERVAL 25 +#define LOG_INTERVAL_STEP 5 +#define LOG_MIN_INTERVAL 5 +#define LOG_MAX_INTERVAL 25 +#define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) -#define LOG_BUF_START(x) ((x)->buffStart) -#define LOG_BUF_END(x) ((x)->buffEnd) -#define LOG_BUF_SIZE(x) ((x)->buffSize) -#define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#define LOG_BUF_START(x) ((x)->buffStart) +#define LOG_BUF_END(x) ((x)->buffEnd) +#define LOG_BUF_SIZE(x) ((x)->buffSize) +#define LOG_BUF_MUTEX(x) ((x)->buffMutex) typedef struct { char *buffer; @@ -48,7 +45,7 @@ typedef struct { int32_t buffEnd; int32_t buffSize; int32_t minBuffSize; - int32_t fd; + TdFilePtr pFile; int32_t stop; pthread_t asyncThread; pthread_mutex_t buffMutex; @@ -67,32 +64,30 @@ typedef struct { pthread_mutex_t logMutex; } SLogObj; -int8_t tscEmbeddedInUtil = 0; +static int8_t tsLogInited = 0; +static SLogObj tsLogObj = {.fileNum = 1}; +int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; -int8_t tsAsyncLog = 1; -float tsTotalLogDirGB = 0; -float tsAvailLogDirGB = 0; -float tsMinimalLogDirGB = 1.0f; -int64_t asyncLogLostLines = 0; -int32_t writeInterval = DEFAULT_LOG_INTERVAL; +bool tsAsyncLog = true; +int32_t tsNumOfLogLines = 10000000; +int64_t tsAsyncLogLostLines = 0; +int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; // log -int32_t tsNumOfLogLines = 10000000; -int32_t mDebugFlag = 131; int32_t dDebugFlag = 135; int32_t vDebugFlag = 135; +int32_t mDebugFlag = 131; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; -int32_t qDebugFlag = 131; -int32_t rpcDebugFlag = 131; +int32_t tmrDebugFlag = 131; int32_t uDebugFlag = 131; -int32_t debugFlag = 0; -int32_t sDebugFlag = 135; +int32_t rpcDebugFlag = 131; +int32_t qDebugFlag = 131; int32_t wDebugFlag = 135; +int32_t sDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t tqDebugFlag = 135; -int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int64_t dbgEmptyW = 0; @@ -101,13 +96,11 @@ int64_t dbgSmallWN = 0; int64_t dbgBigWN = 0; int64_t dbgWSize = 0; -static SLogObj tsLogObj = {.fileNum = 1}; static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); -static void taosCloseLogByFd(int32_t oldFd); +static void taosCloseLogByFd(TdFilePtr pFile); static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); -extern void taosPrintCfg(); static int32_t taosCompressFile(char *srcFileName, char *destFileName); static int32_t taosStartLog() { @@ -120,10 +113,16 @@ static int32_t taosStartLog() { return 0; } -int32_t taosInitLog(char *logName, int numOfLogLines, int maxFiles) { - tsLogObj.logHandle = taosLogBuffNew(TSDB_DEFAULT_LOG_BUF_SIZE); +int32_t taosInitLog(const char *logName, int32_t maxFiles) { + if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; + osUpdate(); + + char fullName[PATH_MAX] = {0}; + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logName); + + tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(logName, numOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; if (taosStartLog() < 0) return -1; return 0; } @@ -136,8 +135,6 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - // tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); - taosMsleep(MAX_LOG_INTERVAL / 1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -147,11 +144,11 @@ void taosCloseLog() { // taosCloseLog(); } -static bool taosLockLogFile(int32_t fd) { - if (fd < 0) return false; +static bool taosLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return false; if (tsLogObj.fileNum > 1) { - int32_t ret = taosLockFile(fd); + int32_t ret = taosLockFile(pFile); if (ret == 0) { return true; } @@ -160,11 +157,11 @@ static bool taosLockLogFile(int32_t fd) { return false; } -static void taosUnLockLogFile(int32_t fd) { - if (fd < 0) return; +static void taosUnLockLogFile(TdFilePtr pFile) { + if (pFile == NULL) return; if (tsLogObj.fileNum > 1) { - taosUnLockFile(fd); + taosUnLockFile(pFile); } } @@ -198,26 +195,25 @@ static void *taosThreadToOpenNewFile(void *param) { taosUmaskFile(0); - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(name, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { tsLogObj.openInProgress = 0; tsLogObj.lines = tsLogObj.maxLines - 1000; - uError("open new log file fail! fd:%d reason:%s, reuse lastlog", fd, strerror(errno)); + uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } - taosLockLogFile(fd); - (void)taosLSeekFile(fd, 0, SEEK_SET); + taosLockLogFile(pFile); + (void)taosLSeekFile(pFile, 0, SEEK_SET); - int32_t oldFd = tsLogObj.logHandle->fd; - tsLogObj.logHandle->fd = fd; + TdFilePtr pOldFile = tsLogObj.logHandle->pFile; + tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosCloseLogByFd(oldFd); + taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); - taosPrintCfg(); taosKeepOldLog(keepName); return NULL; @@ -259,8 +255,8 @@ void taosResetLog() { } static bool taosCheckFileIsOpen(char *logFileName) { - int32_t fd = taosOpenFileWrite(logFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(logFileName, TD_FILE_WRITE); + if (pFile == NULL) { if (errno == ENOENT) { return false; } else { @@ -269,12 +265,12 @@ static bool taosCheckFileIsOpen(char *logFileName) { } } - if (taosLockLogFile(fd)) { - taosUnLockLogFile(fd); - taosCloseFile(fd); + if (taosLockLogFile(pFile)) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); return false; } else { - taosCloseFile(fd); + taosCloseFile(&pFile); return true; } } @@ -347,45 +343,40 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { pthread_mutex_init(&tsLogObj.logMutex, NULL); taosUmaskFile(0); - tsLogObj.logHandle->fd = taosOpenFileCreateWrite(fileName); + tsLogObj.logHandle->pFile = taosOpenFile(fileName, TD_FILE_CTEATE | TD_FILE_WRITE); - if (tsLogObj.logHandle->fd < 0) { + if (tsLogObj.logHandle->pFile == NULL) { printf("\nfailed to open log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - taosLockLogFile(tsLogObj.logHandle->fd); + taosLockLogFile(tsLogObj.logHandle->pFile); // only an estimate for number of lines int64_t filesize = 0; - if (taosFStatFile(tsLogObj.logHandle->fd, &filesize, NULL) < 0) { + if (taosFStatFile(tsLogObj.logHandle->pFile, &filesize, NULL) < 0) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } size = (int32_t)filesize; tsLogObj.lines = size / 60; - taosLSeekFile(tsLogObj.logHandle->fd, 0, SEEK_END); + taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, " new log file \n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); sprintf(name, "==================================================\n"); - taosWriteFile(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name)); + taosWriteFile(tsLogObj.logHandle->pFile, name, (uint32_t)strlen(name)); return 0; } void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; - char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; + char buffer[LOG_MAX_LINE_BUFFER_SIZE] = {0}; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -400,29 +391,29 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - int32_t writeLen = vsnprintf(buffer + len, MAX_LOGLINE_CONTENT_SIZE, format, argpointer); + int32_t writeLen = vsnprintf(buffer + len, LOG_MAX_LINE_CONTENT_SIZE, format, argpointer); if (writeLen <= 0) { - char tmp[MAX_LOGLINE_DUMP_BUFFER_SIZE] = {0}; - writeLen = vsnprintf(tmp, MAX_LOGLINE_DUMP_CONTENT_SIZE, format, argpointer); - strncpy(buffer + len, tmp, MAX_LOGLINE_CONTENT_SIZE); - len += MAX_LOGLINE_CONTENT_SIZE; - } else if (writeLen >= MAX_LOGLINE_CONTENT_SIZE) { - len += MAX_LOGLINE_CONTENT_SIZE; + char tmp[LOG_MAX_LINE_DUMP_BUFFER_SIZE] = {0}; + writeLen = vsnprintf(tmp, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); + strncpy(buffer + len, tmp, LOG_MAX_LINE_CONTENT_SIZE); + len += LOG_MAX_LINE_CONTENT_SIZE; + } else if (writeLen >= LOG_MAX_LINE_CONTENT_SIZE) { + len += LOG_MAX_LINE_CONTENT_SIZE; } else { len += writeLen; } va_end(argpointer); - if (len > MAX_LOGLINE_SIZE) len = MAX_LOGLINE_SIZE; + if (len > LOG_MAX_LINE_SIZE) len = LOG_MAX_LINE_SIZE; buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -432,17 +423,11 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); - if (dflag == 255) nInfo(buffer, len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } void taosDumpData(unsigned char *msg, int32_t len) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; char temp[256]; int32_t i, pos = 0, c = 0; @@ -453,7 +438,7 @@ void taosDumpData(unsigned char *msg, int32_t len) { pos += 3; if (c >= 16) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); c = 0; pos = 0; } @@ -461,19 +446,14 @@ void taosDumpData(unsigned char *msg, int32_t len) { temp[pos++] = '\n'; - taosWriteFile(tsLogObj.logHandle->fd, temp, (uint32_t)pos); + taosWriteFile(tsLogObj.logHandle->pFile, temp, (uint32_t)pos); } void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { - if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, - tsTotalLogDirGB); - fflush(stdout); - return; - } + if (!osLogSpaceAvailable()) return; va_list argpointer; - char buffer[MAX_LOGLINE_DUMP_BUFFER_SIZE]; + char buffer[LOG_MAX_LINE_DUMP_BUFFER_SIZE]; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -488,19 +468,19 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . len += sprintf(buffer + len, "%s", flags); va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_LOGLINE_DUMP_CONTENT_SIZE, format, argpointer); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_CONTENT_SIZE, format, argpointer); va_end(argpointer); - if (len > MAX_LOGLINE_DUMP_SIZE) len = MAX_LOGLINE_DUMP_SIZE; + if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; buffer[len++] = '\n'; buffer[len] = 0; - if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->fd >= 0) { + if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL) { if (tsAsyncLog) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { - taosWriteFile(tsLogObj.logHandle->fd, buffer, len); + taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } if (tsLogObj.maxLines > 0) { @@ -510,19 +490,13 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } } - if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) write(1, buffer, (uint32_t)len); } -#if 0 -void taosCloseLog() { - taosCloseLogByFd(tsLogObj.logHandle->fd); -} -#endif - -static void taosCloseLogByFd(int32_t fd) { - if (fd >= 0) { - taosUnLockLogFile(fd); - taosCloseFile(fd); +static void taosCloseLogByFd(TdFilePtr pFile) { + if (pFile != NULL) { + taosUnLockLogFile(pFile); + taosCloseFile(&pFile); } } @@ -551,15 +525,6 @@ _err: return NULL; } -#if 0 -static void taosLogBuffDestroy(SLogBuff *tLogBuff) { - tsem_destroy(&(tLogBuff->buffNotEmpty)); - pthread_mutex_destroy(&(tLogBuff->buffMutex)); - free(tLogBuff->buffer); - tfree(tLogBuff); -} -#endif - static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, char *msg, int32_t msgLen) { if (start > end) { memcpy(LOG_BUF_BUFFER(tLogBuff) + end, msg, msgLen); @@ -597,7 +562,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) if (remainSize <= msgLen || ((lostLine > 0) && (remainSize <= (msgLen + tmpBufLen)))) { lostLine++; - asyncLogLostLines++; + tsAsyncLogLostLines++; pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); return -1; } @@ -642,13 +607,13 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (start == end) { dbgEmptyW++; - writeInterval = MAX_LOG_INTERVAL; + tsWriteInterval = LOG_MAX_INTERVAL; return; } pollSize = taosGetLogRemainSize(tLogBuff, start, end); if (pollSize < tLogBuff->minBuffSize) { - lastDuration += writeInterval; + lastDuration += tsWriteInterval; if (lastDuration < LOG_MAX_WAIT_MSEC) { break; } @@ -658,12 +623,12 @@ static void taosWriteLog(SLogBuff *tLogBuff) { } if (start < end) { - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->pFile, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; @@ -671,15 +636,15 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (pollSize < tLogBuff->minBuffSize) { dbgSmallWN++; - if (writeInterval < MAX_LOG_INTERVAL) { - writeInterval += LOG_INTERVAL_STEP; + if (tsWriteInterval < LOG_MAX_INTERVAL) { + tsWriteInterval += LOG_INTERVAL_STEP; } } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) { dbgBigWN++; - writeInterval = MIN_LOG_INTERVAL; + tsWriteInterval = LOG_MIN_INTERVAL; } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) { - if (writeInterval > MIN_LOG_INTERVAL) { - writeInterval -= LOG_INTERVAL_STEP; + if (tsWriteInterval > LOG_MIN_INTERVAL) { + tsWriteInterval -= LOG_INTERVAL_STEP; } } @@ -693,7 +658,7 @@ static void taosWriteLog(SLogBuff *tLogBuff) { break; } - writeInterval = MIN_LOG_INTERVAL; + tsWriteInterval = LOG_MIN_INTERVAL; remainChecked = 1; } while (1); @@ -704,7 +669,7 @@ static void *taosAsyncOutputLog(void *param) { setThreadName("log"); while (1) { - taosMsleep(writeInterval); + taosMsleep(tsWriteInterval); // Polling the buffer taosWriteLog(tLogBuff); @@ -720,56 +685,61 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t ret = 0; int32_t len = 0; char *data = malloc(compressSize); - FILE *srcFp = NULL; -// gzFile dstFp = NULL; + // gzFile dstFp = NULL; - srcFp = fopen(srcFileName, "r"); - if (srcFp == NULL) { + // srcFp = fopen(srcFileName, "r"); + TdFilePtr pSrcFile = taosOpenFile(srcFileName, TD_FILE_READ); + if (pSrcFile == NULL) { ret = -1; goto cmp_end; } - int32_t fd = taosOpenFileCreateWriteTrunc(destFileName); - if (fd < 0) { + TdFilePtr pFile = taosOpenFile(destFileName, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { ret = -2; goto cmp_end; } -// dstFp = gzdopen(fd, "wb6f"); -// if (dstFp == NULL) { -// ret = -3; -// close(fd); -// goto cmp_end; -// } -// -// while (!feof(srcFp)) { -// len = (int32_t)fread(data, 1, compressSize, srcFp); -// (void)gzwrite(dstFp, data, len); -// } + // dstFp = gzdopen(fd, "wb6f"); + // if (dstFp == NULL) { + // ret = -3; + // close(fd); + // goto cmp_end; + // } + // + // while (!feof(srcFp)) { + // len = (int32_t)fread(data, 1, compressSize, srcFp); + // (void)gzwrite(dstFp, data, len); + // } cmp_end: - if (srcFp) { - fclose(srcFp); + if (pSrcFile) { + taosCloseFile(&pSrcFile); } -// if (dstFp) { -// gzclose(dstFp); -// } + // if (dstFp) { + // gzclose(dstFp); + // } free(data); return ret; } -void taosPrintOsInfo() { - SysNameInfo info = taosGetSysNameInfo(); - - uInfo(" os pageSize: %" PRId64 "(KB)", tsPageSize); - uInfo(" os openMax: %" PRId64, tsOpenMax); - uInfo(" os streamMax: %" PRId64, tsStreamMax); - uInfo(" os numOfCores: %d", tsNumOfCores); - uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB); - uInfo(" os sysname: %s", info.sysname); - uInfo(" os nodename: %s", info.nodename); - uInfo(" os release: %s", info.release); - uInfo(" os version: %s", info.version); - uInfo(" os machine: %s", info.machine); -} +void taosSetAllDebugFlag(int32_t flag) { + if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + + dDebugFlag = flag; + vDebugFlag = flag; + mDebugFlag = flag; + cDebugFlag = flag; + jniDebugFlag = flag; + uDebugFlag = flag; + rpcDebugFlag = flag; + qDebugFlag = flag; + wDebugFlag = flag; + sDebugFlag = flag; + tsdbDebugFlag = flag; + tqDebugFlag = flag; + fsDebugFlag = flag; + + uInfo("all debug flag are set to %d", flag); +} \ No newline at end of file diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 80bbac2c789e5742854cd9f0fb8b38664fd5e56d..6349ab170c3df30abd94f8b978f16aa598bc22da 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -13,12 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" -#include "ulog.h" +#define _DEFAULT_SOURCE #include "tlosertree.h" #include "taoserror.h" - - +#include "tlog.h" // Set the initial value of the multiway merge tree. static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { @@ -33,10 +31,12 @@ static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { } } -int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, __merge_compare_fn_t compareFn) { +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, + __merge_compare_fn_t compareFn) { int32_t totalEntries = numOfSources << 1u; - SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + SMultiwayMergeTreeInfo* pTreeInfo = + (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); return TAOS_SYSTEM_ERROR(errno); @@ -88,7 +88,7 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { return; } - int32_t parentId = idx >> 1; + int32_t parentId = idx >> 1; STreeNode kLeaf = pTree->pNode[idx]; while (parentId > 0) { diff --git a/source/util/src/mallocator.c b/source/util/src/tmallocator.c similarity index 98% rename from source/util/src/mallocator.c rename to source/util/src/tmallocator.c index a56fbfa5976dfcaaad681ecfda47ad8a1d415a6e..057b908a583f297227982a8f101ab97a7ebe891a 100644 --- a/source/util/src/mallocator.c +++ b/source/util/src/tmallocator.c @@ -13,7 +13,8 @@ * along with this program. If not, see . */ -#include "mallocator.h" +#define _DEFAULT_SOURCE +#include "tmallocator.h" /* ------------------------ HEAP ALLOCATOR ------------------------ */ #if 0 diff --git a/source/util/src/tmd5.c b/source/util/src/tmd5.c index 807f3c8122b31cea64e896aa730529b9af595cf3..2ea4415d7f4327c7a8806b3daebb644bd9a40738 100644 --- a/source/util/src/tmd5.c +++ b/source/util/src/tmd5.c @@ -33,7 +33,7 @@ *********************************************************************** */ -#include "os.h" +#define _DEFAULT_SOURCE #include "tmd5.h" /* forward declaration */ @@ -98,13 +98,13 @@ void tMD5Init(T_MD5_CTX *mdContext) { account for the presence of each of the characters inBuf[0..inLen-1] in the message whose digest is being computed. */ -void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { - uint32_t in[16]; - int mdi; - unsigned int i, ii; +void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, uint32_t inLen) { + uint32_t in[16]; + uint32_t mdi; + uint32_t i, ii; /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + mdi = (uint32_t)((mdContext->i[0] >> 3) & 0x3F); /* update number of bits */ if ((mdContext->i[0] + ((uint32_t)inLen << 3)) < mdContext->i[0]) mdContext->i[1]++; @@ -130,17 +130,17 @@ void tMD5Update(T_MD5_CTX *mdContext, uint8_t *inBuf, unsigned int inLen) { ends with the desired message digest in mdContext->digest[0...15]. */ void tMD5Final(T_MD5_CTX *mdContext) { - uint32_t in[16]; - int mdi; - unsigned int i, ii; - unsigned int padLen; + uint32_t in[16]; + uint32_t mdi; + uint32_t i, ii; + uint32_t padLen; /* save number of bits */ in[14] = mdContext->i[0]; in[15] = mdContext->i[1]; /* compute number of bytes mod 64 */ - mdi = (int)((mdContext->i[0] >> 3) & 0x3F); + mdi = (uint32_t)((mdContext->i[0] >> 3) & 0x3F); /* pad out to 56 mod 64 */ padLen = (mdi < 56) ? (56 - mdi) : (120 - mdi); diff --git a/source/util/src/tmempool.c b/source/util/src/tmempool.c index a61c4abb146ec7497fca7499ac06773812cdea90..1fc9bfc7ab458dfda7d1e74123bbea2366a70e38 100644 --- a/source/util/src/tmempool.c +++ b/source/util/src/tmempool.c @@ -13,23 +13,23 @@ * along with this program. If not, see . */ -#include "os.h" -#include "ulog.h" +#define _DEFAULT_SOURCE #include "tmempool.h" +#include "tlog.h" #include "tutil.h" typedef struct { - int numOfFree; /* number of free slots */ - int first; /* the first free slot */ - int numOfBlock; /* the number of blocks */ - int blockSize; /* block size in bytes */ - int * freeList; /* the index list */ - char * pool; /* the actual mem block */ + int32_t numOfFree; /* number of free slots */ + int32_t first; /* the first free slot */ + int32_t numOfBlock; /* the number of blocks */ + int32_t blockSize; /* block size in bytes */ + int32_t *freeList; /* the index list */ + char *pool; /* the actual mem block */ pthread_mutex_t mutex; } pool_t; -mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { - int i; +mpool_h taosMemPoolInit(int32_t numOfBlock, int32_t blockSize) { + int32_t i; pool_t *pool_p; if (numOfBlock <= 1 || blockSize <= 1) { @@ -48,7 +48,7 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { pool_p->blockSize = blockSize; pool_p->numOfBlock = numOfBlock; pool_p->pool = (char *)malloc((size_t)(blockSize * numOfBlock)); - pool_p->freeList = (int *)malloc(sizeof(int) * (size_t)numOfBlock); + pool_p->freeList = (int32_t *)malloc(sizeof(int32_t) * (size_t)numOfBlock); if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); @@ -70,7 +70,7 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { } char *taosMemPoolMalloc(mpool_h handle) { - char * pos = NULL; + char *pos = NULL; pool_t *pool_p = (pool_t *)handle; pthread_mutex_lock(&(pool_p->mutex)); @@ -89,20 +89,20 @@ char *taosMemPoolMalloc(mpool_h handle) { } void taosMemPoolFree(mpool_h handle, char *pMem) { - int index; + int32_t index; pool_t *pool_p = (pool_t *)handle; if (pMem == NULL) return; - index = (int)(pMem - pool_p->pool) % pool_p->blockSize; + index = (int32_t)(pMem - pool_p->pool) % pool_p->blockSize; if (index != 0) { uError("invalid free address:%p\n", pMem); return; } - index = (int)((pMem - pool_p->pool) / pool_p->blockSize); + index = (int32_t)((pMem - pool_p->pool) / pool_p->blockSize); if (index < 0 || index >= pool_p->numOfBlock) { - uError("mempool: error, invalid address:%p\n", pMem); + uError("mempool: error, invalid address:%p", pMem); return; } diff --git a/source/util/src/tnote.c b/source/util/src/tnote.c deleted file mode 100644 index d9356b7e4049cfdf9bc77ff1606e24993758bd6c..0000000000000000000000000000000000000000 --- a/source/util/src/tnote.c +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tutil.h" -#include "tdef.h" -#include "tnote.h" - -SNoteObj tsHttpNote; -SNoteObj tsTscNote; -SNoteObj tsInfoNote; - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote); -static void taosCloseNoteByFd(int32_t oldFd, SNoteObj *pNote); - -static void taosInitNote(int32_t numOfLines, int32_t maxNotes, SNoteObj *pNote, char *name) { - memset(pNote, 0, sizeof(SNoteObj)); - pNote->fileNum = 1; - pNote->fd = -1; - - if (taosOpenNoteWithMaxLines(name, numOfLines, maxNotes, pNote) < 0) { - fprintf(stderr, "failed to init note file\n"); - } - - taosNotePrint(pNote, "=================================================="); - taosNotePrint(pNote, "=================== new note ==================="); - taosNotePrint(pNote, "=================================================="); -} - -int32_t taosInitNotes() { - char name[TSDB_FILENAME_LEN * 2] = {0}; - -#if 0 - if (tsTscEnableRecordSql) { - snprintf(name, TSDB_FILENAME_LEN * 2, "%s/tscsql-%d", tsLogDir, taosGetPId()); - taosInitNote(tsNumOfLogLines, 1, &tsTscNote, name); - } - -#endif - return 0; -} - -static bool taosLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return false; - - if (pNote->fileNum > 1) { - int32_t ret = (int32_t)taosLockFile(fd); - if (ret == 0) { - return true; - } - } - - return false; -} - -static void taosUnLockNote(int32_t fd, SNoteObj *pNote) { - if (fd < 0) return; - - if (pNote->fileNum > 1) { - taosUnLockFile(fd); - } -} - -static void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN * 2]; - SNoteObj *pNote = (SNoteObj *)param; - - setThreadName("openNewNote"); - - pNote->flag ^= 1; - pNote->lines = 0; - sprintf(name, "%s.%d", pNote->name, pNote->flag); - - taosUmaskFile(0); - - int32_t fd = taosOpenFileCreateWriteTrunc(name); - if (fd < 0) { - return NULL; - } - - taosLockNote(fd, pNote); - (void)taosLSeekFile(fd, 0, SEEK_SET); - - int32_t oldFd = pNote->fd; - pNote->fd = fd; - pNote->lines = 0; - pNote->openInProgress = 0; - taosNotePrint(pNote, "=============== new note is opened ============="); - - taosCloseNoteByFd(oldFd, pNote); - return NULL; -} - -static int32_t taosOpenNewNote(SNoteObj *pNote) { - pthread_mutex_lock(&pNote->mutex); - - if (pNote->lines > pNote->maxLines && pNote->openInProgress == 0) { - pNote->openInProgress = 1; - - taosNotePrint(pNote, "=============== open new note =================="); - pthread_t pattern; - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - pthread_create(&pattern, &attr, taosThreadToOpenNewNote, (void *)pNote); - pthread_attr_destroy(&attr); - } - - pthread_mutex_unlock(&pNote->mutex); - - return pNote->fd; -} - -static bool taosCheckNoteIsOpen(char *noteName, SNoteObj *pNote) { - int32_t fd = taosOpenFileCreateWrite(noteName); - if (fd < 0) { - fprintf(stderr, "failed to open note:%s reason:%s\n", noteName, strerror(errno)); - return true; - } - - if (taosLockNote(fd, pNote)) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - return false; - } else { - taosCloseFile(fd); - return true; - } -} - -static void taosGetNoteName(char *fn, SNoteObj *pNote) { - if (pNote->fileNum > 1) { - for (int32_t i = 0; i < pNote->fileNum; i++) { - char fileName[NOTE_FILE_NAME_LEN]; - - sprintf(fileName, "%s%d.0", fn, i); - bool file1open = taosCheckNoteIsOpen(fileName, pNote); - - sprintf(fileName, "%s%d.1", fn, i); - bool file2open = taosCheckNoteIsOpen(fileName, pNote); - - if (!file1open && !file2open) { - sprintf(pNote->name, "%s%d", fn, i); - return; - } - } - } - - if (strlen(fn) < NOTE_FILE_NAME_LEN) { - strcpy(pNote->name, fn); - } -} - -static int32_t taosOpenNoteWithMaxLines(char *fn, int32_t maxLines, int32_t maxNoteNum, SNoteObj *pNote) { - char name[NOTE_FILE_NAME_LEN * 2] = {0}; - int32_t size; - int32_t logstat0_mtime, logstat1_mtime; - - pNote->maxLines = maxLines; - pNote->fileNum = maxNoteNum; - taosGetNoteName(fn, pNote); - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".0"); - } - bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; - - if (strlen(fn) < NOTE_FILE_NAME_LEN + 50 - 2) { - strcpy(name, fn); - strcat(name, ".1"); - } - bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; - - if (!log0Exist && !log1Exist) { - pNote->flag = 0; - } else if (!log1Exist) { - pNote->flag = 0; - } else if (!log0Exist) { - pNote->flag = 1; - } else { - pNote->flag = (logstat0_mtime > logstat1_mtime) ? 0 : 1; - } - - char noteName[NOTE_FILE_NAME_LEN * 2] = {0}; - sprintf(noteName, "%s.%d", pNote->name, pNote->flag); - pthread_mutex_init(&pNote->mutex, NULL); - - taosUmaskFile(0); - pNote->fd = taosOpenFileCreateWrite(noteName); - - if (pNote->fd < 0) { - fprintf(stderr, "failed to open note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - taosLockNote(pNote->fd, pNote); - - // only an estimate for number of lines - int64_t filestat_size; - if (taosFStatFile(pNote->fd, &filestat_size, NULL) < 0) { - fprintf(stderr, "failed to fstat note file:%s reason:%s\n", noteName, strerror(errno)); - return -1; - } - size = (int32_t)filestat_size; - pNote->lines = size / 60; - - taosLSeekFile(pNote->fd, 0, SEEK_END); - - return 0; -} - -void taosNotePrintBuffer(SNoteObj *pNote, char *buffer, int32_t len) { - if (pNote->fd <= 0) return; - taosWriteFile(pNote->fd, buffer, len); - - if (pNote->maxLines > 0) { - pNote->lines++; - if ((pNote->lines > pNote->maxLines) && (pNote->openInProgress == 0)) taosOpenNewNote(pNote); - } -} - -void taosNotePrint(SNoteObj *pNote, const char *const format, ...) { - va_list argpointer; - char buffer[MAX_NOTE_LINE_SIZE + 2]; - int32_t len; - struct tm Tm, *ptm; - struct timeval timeSecs; - time_t curTime; - - taosGetTimeOfDay(&timeSecs); - curTime = timeSecs.tv_sec; - ptm = localtime_r(&curTime, &Tm); - len = sprintf(buffer, "%02d/%02d %02d:%02d:%02d.%06d %08" PRId64 " ", ptm->tm_mon + 1, ptm->tm_mday, ptm->tm_hour, - ptm->tm_min, ptm->tm_sec, (int32_t)timeSecs.tv_usec, taosGetSelfPthreadId()); - va_start(argpointer, format); - len += vsnprintf(buffer + len, MAX_NOTE_LINE_SIZE - len, format, argpointer); - va_end(argpointer); - - if (len >= MAX_NOTE_LINE_SIZE) len = MAX_NOTE_LINE_SIZE - 2; - - buffer[len++] = '\n'; - buffer[len] = 0; - - taosNotePrintBuffer(pNote, buffer, len); -} - -// static void taosCloseNote(SNoteObj *pNote) { taosCloseNoteByFd(pNote->fd, pNote); } - -static void taosCloseNoteByFd(int32_t fd, SNoteObj *pNote) { - if (fd >= 0) { - taosUnLockNote(fd, pNote); - taosCloseFile(fd); - } -} diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 0e8d85492ce8811448ea806ebcac942a52a1bb26..45c38165a11c47717b740fd74e5f43a5947c9406 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -1,109 +1,64 @@ -#include "os.h" -#include "ulog.h" +#define _DEFAULT_SOURCE #include "tpagedbuf.h" #include "taoserror.h" #include "tcompression.h" #include "thash.h" +#include "tlog.h" -#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) +#define GET_DATA_PAYLOAD(_p) ((char*)(_p)->pData + POINTER_BYTES) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) -typedef struct SFreeListItem { - int32_t offset; - int32_t len; -} SFreeListItem; - typedef struct SPageDiskInfo { - int64_t offset; - int32_t length; -} SPageDiskInfo; - -typedef struct SPageInfo { - SListNode* pn; // point to list node - void* pData; - int64_t offset; - int32_t pageId; - int32_t length:30; - bool used:1; // set current page is in used - bool dirty:1; // set current buffer page is dirty or not -} SPageInfo; - -typedef struct SDiskbasedBuf { + int64_t offset; + int32_t length; +} SPageDiskInfo, SFreeListItem; + +struct SPageInfo { + SListNode* pn; // point to list node struct + void* pData; + int64_t offset; + int32_t pageId; + int32_t length : 29; + bool used : 1; // set current page is in used + bool dirty : 1; // set current buffer page is dirty or not +}; + +struct SDiskbasedBuf { int32_t numOfPages; int64_t totalBufSize; - uint64_t fileSize; // disk file size - FILE* file; - int32_t allocateId; // allocated page id - char* path; // file path - int32_t pageSize; // current used page size - int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* groupSet; // id hash table + uint64_t fileSize; // disk file size + TdFilePtr pFile; + int32_t allocateId; // allocated page id + char* path; // file path + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory + SList* freePgList; // free page list + SHashObj* groupSet; // id hash table, todo remove it SHashObj* all; SList* lruList; - void* emptyDummyIdList; // dummy id list - void* assistBuf; // assistant buffer for compress/decompress data - SArray* pFree; // free area in file - bool comp; // compressed before flushed to disk - uint64_t nextPos; // next page flush position - - uint64_t qId; // for debug purpose - bool printStatis; // Print statistics info when closing this buffer. + void* emptyDummyIdList; // dummy id list + void* assistBuf; // assistant buffer for compress/decompress data + SArray* pFree; // free area in file + bool comp; // compressed before flushed to disk + uint64_t nextPos; // next page flush position + + uint64_t qId; // for debug purpose + bool printStatis; // Print statistics info when closing this buffer. SDiskbasedBufStatis statis; -} SDiskbasedBuf; - -static void printStatisData(const SDiskbasedBuf* pBuf); - - int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) { - *pBuf = calloc(1, sizeof(SDiskbasedBuf)); - - SDiskbasedBuf* pResBuf = *pBuf; - if (pResBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = 0; // all pages are in buffer in the first place - pResBuf->totalBufSize = 0; - pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. - pResBuf->allocateId = -1; - pResBuf->comp = true; - pResBuf->file = NULL; - pResBuf->qId = qId; - pResBuf->fileSize = 0; - - // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); - - pResBuf->lruList = tdListNew(POINTER_BYTES); - - // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - -// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, -// pResBuf->inMemPages, pResBuf->path); - - return TSDB_CODE_SUCCESS; -} +}; static int32_t createDiskFile(SDiskbasedBuf* pBuf) { - pBuf->file = fopen(pBuf->path, "wb+"); - if (pBuf->file == NULL) { -// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); + // pBuf->file = fopen(pBuf->path, "wb+"); + pBuf->pFile = taosOpenFile(pBuf->path, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (pBuf->pFile == NULL) { + // qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } return TSDB_CODE_SUCCESS; } -static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing +static char* doCompressData(void* data, int32_t srcSize, int32_t* dst, SDiskbasedBuf* pBuf) { // do nothing if (!pBuf->comp) { *dst = srcSize; return data; @@ -115,13 +70,13 @@ static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbase return data; } -static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing +static char* doDecompressData(void* data, int32_t srcSize, int32_t* dst, SDiskbasedBuf* pBuf) { // do nothing if (!pBuf->comp) { *dst = srcSize; return data; } - *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0); + *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize, ONE_STAGE_COMP, NULL, 0); if (*dst > 0) { memcpy(data, pBuf->assistBuf, *dst); } @@ -135,12 +90,12 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) { int32_t offset = -1; size_t num = taosArrayGetSize(pBuf->pFree); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { SFreeListItem* pi = taosArrayGet(pBuf->pFree, i); - if (pi->len >= size) { + if (pi->length >= size) { offset = pi->offset; pi->offset += (int32_t)size; - pi->len -= (int32_t)size; + pi->length -= (int32_t)size; return offset; } @@ -151,108 +106,131 @@ static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) { } } +static void setPageNotInBuf(SPageInfo* pPageInfo) { pPageInfo->pData = NULL; } + +static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { return pageSize + POINTER_BYTES + 2; } + +/** + * +--------------------------+-------------------+--------------+ + * | PTR to SPageInfo (8bytes)| Payload (PageSize)| 2 Extra Bytes| + * +--------------------------+-------------------+--------------+ + * @param pBuf + * @param pg + * @return + */ static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { assert(!pg->used && pg->pData != NULL); - int32_t size = -1; + int32_t size = pBuf->pageSize; char* t = NULL; if (pg->offset == -1 || pg->dirty) { - SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - t = doCompressData(pPage->data, pBuf->pageSize, &size, pBuf); + void* payload = GET_DATA_PAYLOAD(pg); + t = doCompressData(payload, pBuf->pageSize, &size, pBuf); + assert(size >= 0); } // this page is flushed to disk for the first time - if (pg->offset == -1) { - assert(pg->dirty == true); + if (pg->dirty) { + if (pg->offset == -1) { + assert(pg->dirty == true); - pg->offset = allocatePositionInFile(pBuf, size); - pBuf->nextPos += size; + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - ret = (int32_t) fwrite(t, 1, size, pBuf->file); - if (ret != size) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + ret = (int32_t)taosWriteFile(pBuf->pFile, t, size); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - if (pBuf->fileSize < pg->offset + size) { - pBuf->fileSize = pg->offset + size; - } + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } - pBuf->statis.flushBytes += size; - pBuf->statis.flushPages += 1; - } else if (pg->dirty) { - // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing - if (pg->length < size) { - // 1. add current space to free list - SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset}; - taosArrayPush(pBuf->pFree, &dinfo); + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } else { + // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing + if (pg->length < size) { + // 1. add current space to free list + SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset}; + taosArrayPush(pBuf->pFree, &dinfo); + + // 2. allocate new position, and update the info + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; + } - // 2. allocate new position, and update the info - pg->offset = allocatePositionInFile(pBuf, size); - pBuf->nextPos += size; - } + // 3. write to disk. + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - // 3. write to disk. - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); - if (ret != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + ret = (int32_t)taosWriteFile(pBuf->pFile, t, size); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } - ret = (int32_t)fwrite(t, 1, size, pBuf->file); - if (ret != size) { - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } - if (pBuf->fileSize < pg->offset + size) { - pBuf->fileSize = pg->offset + size; + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } + } else { // NOTE: the size may be -1, the this recycle page has not been flushed to disk yet. + size = pg->length; + if (size == -1) { + printf("----\n"); } - - pBuf->statis.flushBytes += size; - pBuf->statis.flushPages += 1; } - char* pDataBuf = pg->pData; - memset(pDataBuf, 0, pBuf->pageSize + sizeof(SFilePage)); + ASSERT(size > 0 || (pg->offset == -1 && pg->length == -1)); - pg->pData = NULL; // this means the data is not in buffer - pg->length = size; - pg->dirty = false; + char* pDataBuf = pg->pData; + memset(pDataBuf, 0, getAllocPageSize(pBuf->pageSize)); + pg->length = size; // on disk size return pDataBuf; } static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); + assert(((int64_t)pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); - if (pBuf->file == NULL) { + if (pBuf->pFile == NULL) { if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { terrno = ret; return NULL; } } - return doFlushPageToDisk(pBuf, pg); + char* p = doFlushPageToDisk(pBuf, pg); + setPageNotInBuf(pg); + pg->dirty = false; + + return p; } // load file block data in disk static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { - int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + int32_t ret = taosLSeekFile(pBuf->pFile, pg->offset, SEEK_SET); if (ret != 0) { ret = TAOS_SYSTEM_ERROR(errno); return ret; } - SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); - ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + void* pPage = (void*)GET_DATA_PAYLOAD(pg); + ret = (int32_t)taosReadFile(pBuf->pFile, pPage, pg->length); if (ret != pg->length) { ret = TAOS_SYSTEM_ERROR(errno); return ret; @@ -262,12 +240,12 @@ static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { pBuf->statis.loadPages += 1; int32_t fullSize = 0; - doDecompressData(pPage->data, pg->length, &fullSize, pBuf); + doDecompressData(pPage, pg->length, &fullSize, pBuf); return 0; } static SIDList addNewGroup(SDiskbasedBuf* pBuf, int32_t groupId) { - assert(taosHashGet(pBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); + assert(taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t)) == NULL); SArray* pa = taosArrayInit(1, POINTER_BYTES); int32_t ret = taosHashPut(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); @@ -283,21 +261,21 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag if (p == NULL) { // it is a new group id list = addNewGroup(pBuf, groupId); } else { - list = (SIDList) (*p); + list = (SIDList)(*p); } pBuf->numOfPages += 1; - SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; + SPageInfo* ppi = malloc(sizeof(SPageInfo)); ppi->pageId = pageId; - ppi->pData = NULL; + ppi->pData = NULL; ppi->offset = -1; ppi->length = -1; - ppi->used = true; - ppi->pn = NULL; + ppi->used = true; + ppi->pn = NULL; - return *(SPageInfo**) taosArrayPush(list, &ppi); + return *(SPageInfo**)taosArrayPush(list, &ppi); } static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { @@ -305,22 +283,33 @@ static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { tdListInitIter(pBuf->lruList, &iter, TD_LIST_BACKWARD); SListNode* pn = NULL; - while((pn = tdListNext(&iter)) != NULL) { - assert(pn != NULL); - - SPageInfo* pageInfo = *(SPageInfo**) pn->data; + while ((pn = tdListNext(&iter)) != NULL) { + SPageInfo* pageInfo = *(SPageInfo**)pn->data; assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); if (!pageInfo->used) { + // printf("%d is chosen\n", pageInfo->pageId); break; + } else { + // printf("page %d is used, dirty:%d\n", pageInfo->pageId, pageInfo->dirty); } } + // int32_t pos = listNEles(pBuf->lruList); + // SListIter iter1 = {0}; + // tdListInitIter(pBuf->lruList, &iter1, TD_LIST_BACKWARD); + // SListNode* pn1 = NULL; + // while((pn1 = tdListNext(&iter1)) != NULL) { + // SPageInfo* pageInfo = *(SPageInfo**) pn1->data; + // printf("page %d is used, dirty:%d, pos:%d\n", pageInfo->pageId, pageInfo->dirty, pos - 1); + // pos -= 1; + // } + return pn; } static char* evacOneDataPage(SDiskbasedBuf* pBuf) { - char* bufPage = NULL; + char* bufPage = NULL; SListNode* pn = getEldestUnrefedPage(pBuf); // all pages are referenced by user, try to allocate new space @@ -331,12 +320,12 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { // increase by 50% of previous mem pages pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f); -// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev, -// pBuf->inMemPages, pBuf->pageSize); + // qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev, + // pBuf->inMemPages, pBuf->pageSize); } else { tdListPopNode(pBuf->lruList, pn); - SPageInfo* d = *(SPageInfo**) pn->data; + SPageInfo* d = *(SPageInfo**)pn->data; assert(d->pn == pn); d->pn = NULL; @@ -348,22 +337,71 @@ static char* evacOneDataPage(SDiskbasedBuf* pBuf) { return bufPage; } -static void lruListPushFront(SList *pList, SPageInfo* pi) { +static void lruListPushFront(SList* pList, SPageInfo* pi) { tdListPrepend(pList, &pi); SListNode* front = tdListGetHead(pList); pi->pn = front; } -static void lruListMoveToFront(SList *pList, SPageInfo* pi) { +static void lruListMoveToFront(SList* pList, SPageInfo* pi) { tdListPopNode(pList, pi->pn); tdListPrependNode(pList, pi->pn); } -static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { - return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage); +static SPageInfo* getPageInfoFromPayload(void* page) { + int32_t offset = offsetof(SPageInfo, pData); + char* p = page - offset; + + SPageInfo* ppi = ((SPageInfo**)p)[0]; + return ppi; +} + +int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, + const char* dir) { + *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + + SDiskbasedBuf* pPBuf = *pBuf; + if (pPBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pPBuf->pageSize = pagesize; + pPBuf->numOfPages = 0; // all pages are in buffer in the first place + pPBuf->totalBufSize = 0; + pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit. + pPBuf->allocateId = -1; + pPBuf->comp = true; + pPBuf->pFile = NULL; + pPBuf->qId = qId; + pPBuf->fileSize = 0; + pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem)); + pPBuf->freePgList = tdListNew(POINTER_BYTES); + + // at least more than 2 pages must be in memory + assert(inMemBufSize >= pagesize * 2); + + pPBuf->lruList = tdListNew(POINTER_BYTES); + + // init id hash table + _hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + pPBuf->groupSet = taosHashInit(10, fn, true, false); + pPBuf->assistBuf = malloc(pPBuf->pageSize + 2); // EXTRA BYTES + pPBuf->all = taosHashInit(10, fn, true, false); + + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(dir, "paged-buf", path); + pPBuf->path = strdup(path); + + pPBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); + + // qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, + // pPBuf->pageSize, + // pPBuf->inMemPages, pPBuf->path); + + return TSDB_CODE_SUCCESS; } -SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { +void* getNewBufPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { pBuf->statis.getPages += 1; char* availablePage = NULL; @@ -372,23 +410,34 @@ SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) // Failed to allocate a new buffer page, and there is an error occurs. if (availablePage == NULL) { + assert(0); return NULL; } } - // register new id in this group - *pageId = (++pBuf->allocateId); - - // register page id info - SPageInfo* pi = registerPage(pBuf, groupId, *pageId); + SPageInfo* pi = NULL; + if (listNEles(pBuf->freePgList) != 0) { + SListNode* pItem = tdListPopHead(pBuf->freePgList); + pi = *(SPageInfo**)pItem->data; + pi->used = true; + *pageId = pi->pageId; + tfree(pItem); + } else { // create a new pageinfo + // register new id in this group + *pageId = (++pBuf->allocateId); + + // register page id info + pi = registerPage(pBuf, groupId, *pageId); + + // add to hash map + taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); + pBuf->totalBufSize += pBuf->pageSize; + } // add to LRU list assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0); lruListPushFront(pBuf->lruList, pi); - // add to hash map - taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); - // allocate buf if (availablePage == NULL) { pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. @@ -396,37 +445,32 @@ SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) pi->pData = availablePage; } - pBuf->totalBufSize += pBuf->pageSize; - ((void**)pi->pData)[0] = pi; - pi->used = true; - - return (void *)(GET_DATA_PAYLOAD(pi)); + return (void*)(GET_DATA_PAYLOAD(pi)); } -SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { +void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { assert(pBuf != NULL && id >= 0); pBuf->statis.getPages += 1; SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t)); assert(pi != NULL && *pi != NULL); - if ((*pi)->pData != NULL) { // it is in memory + if ((*pi)->pData != NULL) { // it is in memory // no need to update the LRU list if only one page exists if (pBuf->numOfPages == 1) { (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); + return (void*)(GET_DATA_PAYLOAD(*pi)); } - SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); + SPageInfo** pInfo = (SPageInfo**)((*pi)->pn->data); assert(*pInfo == *pi); lruListMoveToFront(pBuf->lruList, (*pi)); (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); - - } else { // not in memory + return (void*)(GET_DATA_PAYLOAD(*pi)); + } else { // not in memory assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0); char* availablePage = NULL; @@ -443,6 +487,7 @@ SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { (*pi)->pData = availablePage; } + // set the ptr to the new SPageInfo ((void**)((*pi)->pData))[0] = (*pi); lruListPushFront(pBuf->lruList, *pi); @@ -453,27 +498,24 @@ SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { return NULL; } - return (void *)(GET_DATA_PAYLOAD(*pi)); + return (void*)(GET_DATA_PAYLOAD(*pi)); } } void releaseBufPage(SDiskbasedBuf* pBuf, void* page) { assert(pBuf != NULL && page != NULL); - int32_t offset = offsetof(SPageInfo, pData); - char* p = page - offset; - - SPageInfo* ppi = ((SPageInfo**) p)[0]; + SPageInfo* ppi = getPageInfoFromPayload(page); releaseBufPageInfo(pBuf, ppi); } void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { - assert(pi->pData != NULL && pi->used); + assert(pi->pData != NULL && pi->used == true); pi->used = false; pBuf->statis.releasePages += 1; } -size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); } +size_t getNumOfBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); } size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; } @@ -484,42 +526,46 @@ SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId) { if (p == NULL) { // it is a new group id return pBuf->emptyDummyIdList; } else { - return (SArray*) (*p); + return (SArray*)(*p); } } -void destroyResultBuf(SDiskbasedBuf* pBuf) { +void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { if (pBuf == NULL) { return; } - printStatisData(pBuf); + dBufPrintStatis(pBuf); - if (pBuf->file != NULL) { - uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", - pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, - listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); + if (pBuf->pFile != NULL) { + uDebug( + "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page " + "size:%.2f Kb, %" PRIx64 "\n", + pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); - fclose(pBuf->file); + taosCloseFile(&pBuf->pFile); } else { - uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); + uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId); } // print the statistics information { - SDiskbasedBufStatis *ps = &pBuf->statis; - uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n" - , ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages - , ps->loadBytes/(1024.0 * ps->loadPages)); + SDiskbasedBufStatis* ps = &pBuf->statis; + uDebug( + "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f " + "Kb\n", + ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f, + ps->loadPages, ps->loadBytes / (1024.0 * ps->loadPages)); } remove(pBuf->path); tfree(pBuf->path); SArray** p = taosHashIterate(pBuf->groupSet, NULL); - while(p) { + while (p) { size_t n = taosArrayGetSize(*p); - for(int32_t i = 0; i < n; ++i) { + for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); tfree(pi->pData); tfree(pi); @@ -530,7 +576,11 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { } tdListFree(pBuf->lruList); + tdListFree(pBuf->freePgList); + taosArrayDestroy(pBuf->emptyDummyIdList); + taosArrayDestroy(pBuf->pFree); + taosHashCleanup(pBuf->groupSet); taosHashCleanup(pBuf->all); @@ -539,7 +589,7 @@ void destroyResultBuf(SDiskbasedBuf* pBuf) { } SPageInfo* getLastPageInfo(SIDList pList) { - size_t size = taosArrayGetSize(pList); + size_t size = taosArrayGetSize(pList); SPageInfo* pPgInfo = taosArrayGetP(pList, size - 1); return pPgInfo; } @@ -549,35 +599,38 @@ int32_t getPageId(const SPageInfo* pPgInfo) { return pPgInfo->pageId; } -int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { - return pBuf->pageSize; -} - -int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { - return pBuf->inMemPages; -} +int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { return pBuf->pageSize; } -bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { - return pBuf->fileSize == 0; -} +int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { return pBuf->inMemPages; } -void setBufPageDirty(SFilePage* pPage, bool dirty) { - int32_t offset = offsetof(SPageInfo, pData); // todo extract method - char* p = (char*)pPage - offset; +bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { return pBuf->fileSize == 0; } - SPageInfo* ppi = ((SPageInfo**) p)[0]; +void setBufPageDirty(void* pPage, bool dirty) { + SPageInfo* ppi = getPageInfoFromPayload(pPage); ppi->dirty = dirty; } -void printStatisBeforeClose(SDiskbasedBuf* pBuf) { - pBuf->printStatis = true; -} +void setBufPageCompressOnDisk(SDiskbasedBuf* pBuf, bool comp) { pBuf->comp = comp; } + +void dBufSetBufPageRecycled(SDiskbasedBuf* pBuf, void* pPage) { + SPageInfo* ppi = getPageInfoFromPayload(pPage); + + ppi->used = false; + ppi->dirty = false; -SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { - return pBuf->statis; + // add this pageinfo into the free page info list + SListNode* pNode = tdListPopNode(pBuf->lruList, ppi->pn); + tfree(ppi->pData); + tfree(pNode); + + tdListAppend(pBuf->freePgList, &ppi); } -void printStatisData(const SDiskbasedBuf* pBuf) { +void dBufSetPrintInfo(SDiskbasedBuf* pBuf) { pBuf->printStatis = true; } + +SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { return pBuf->statis; } + +void dBufPrintStatis(const SDiskbasedBuf* pBuf) { if (!pBuf->printStatis) { return; } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 8125f550d05df47d6f1b9880ee82c82e83a307c0..000352203348ec8ea0ff3d587c8de89557d994dd 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "tqueue.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef struct STaosQnode STaosQnode; @@ -30,19 +30,19 @@ typedef struct STaosQueue { int32_t itemSize; int32_t numOfItems; int32_t threadId; - STaosQnode * head; - STaosQnode * tail; - STaosQueue * next; // for queue set - STaosQset * qset; // for queue set - void * ahandle; // for queue set + STaosQnode *head; + STaosQnode *tail; + STaosQueue *next; // for queue set + STaosQset *qset; // for queue set + void *ahandle; // for queue set FItem itemFp; FItems itemsFp; pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { - STaosQueue * head; - STaosQueue * current; + STaosQueue *head; + STaosQueue *current; pthread_mutex_t mutex; int32_t numOfQueues; int32_t numOfItems; @@ -82,7 +82,7 @@ void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset * qset; + STaosQset *qset; pthread_mutex_lock(&queue->mutex); STaosQnode *pNode = queue->head; diff --git a/source/util/src/tref.c b/source/util/src/tref.c index 68f161bd3dc6c12f121855c6edf04f62a2f285cb..a9f6c21bf80ad48f34c895e25c46074fe013d7cd 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" +#define _DEFAULT_SOURCE +#include "tref.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" #include "tutil.h" #define TSDB_REF_OBJECTS 50 @@ -24,49 +25,48 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; // previous node - struct SRefNode *next; // next node - void *p; // pointer to resource protected, - int64_t rid; // reference ID - int32_t count; // number of references - int removed; // 1: removed + struct SRefNode *prev; // previous node + struct SRefNode *next; // next node + void *p; // pointer to resource protected, + int64_t rid; // reference ID + int32_t count; // number of references + int32_t removed; // 1: removed } SRefNode; - + typedef struct { - SRefNode **nodeList; // array of SRefNode linked list - int state; // 0: empty, 1: active; 2: deleted - int rsetId; // refSet ID, global unique - int64_t rid; // increase by one for each new reference - int max; // mod - int32_t count; // total number of SRefNodes in this set + SRefNode **nodeList; // array of SRefNode linked list + int32_t state; // 0: empty, 1: active; 2: deleted + int32_t rsetId; // refSet ID, global unique + int64_t rid; // increase by one for each new reference + int32_t max; // mod + int32_t count; // total number of SRefNodes in this set int64_t *lockedBy; - void (*fp)(void *); + void (*fp)(void *); } SRefSet; static SRefSet tsRefSetList[TSDB_REF_OBJECTS]; static pthread_once_t tsRefModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tsRefMutex; -static int tsRefSetNum = 0; -static int tsNextId = 0; - -static void taosInitRefModule(void); -static void taosLockList(int64_t *lockedBy); -static void taosUnlockList(int64_t *lockedBy); -static void taosIncRsetCount(SRefSet *pSet); -static void taosDecRsetCount(SRefSet *pSet); -static int taosDecRefCount(int rsetId, int64_t rid, int remove); - -int taosOpenRef(int max, void (*fp)(void *)) -{ +static int32_t tsRefSetNum = 0; +static int32_t tsNextId = 0; + +static void taosInitRefModule(void); +static void taosLockList(int64_t *lockedBy); +static void taosUnlockList(int64_t *lockedBy); +static void taosIncRsetCount(SRefSet *pSet); +static void taosDecRsetCount(SRefSet *pSet); +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove); + +int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { SRefNode **nodeList; SRefSet *pSet; int64_t *lockedBy; - int i, rsetId; + int32_t i, rsetId; pthread_once(&tsRefModuleInit, taosInitRefModule); nodeList = calloc(sizeof(SRefNode *), (size_t)max); - if (nodeList == NULL) { + if (nodeList == NULL) { terrno = TSDB_CODE_REF_NO_MEMORY; return -1; } @@ -82,7 +82,7 @@ int taosOpenRef(int max, void (*fp)(void *)) for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; - if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId + if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; } @@ -102,8 +102,8 @@ int taosOpenRef(int max, void (*fp)(void *)) uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { rsetId = TSDB_CODE_REF_FULL; - free (nodeList); - free (lockedBy); + free(nodeList); + free(lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); } @@ -112,10 +112,9 @@ int taosOpenRef(int max, void (*fp)(void *)) return rsetId; } -int taosCloseRef(int rsetId) -{ - SRefSet *pSet; - int deleted = 0; +int32_t taosCloseRef(int32_t rsetId) { + SRefSet *pSet; + int32_t deleted = 0; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { uTrace("rsetId:%d is invalid, out of range", rsetId); @@ -142,9 +141,8 @@ int taosCloseRef(int rsetId) return 0; } -int64_t taosAddRef(int rsetId, void *p) -{ - int hash; +int64_t taosAddRef(int32_t rsetId, void *p) { + int32_t hash; SRefNode *pNode; SRefSet *pSet; int64_t rid = 0; @@ -172,7 +170,7 @@ int64_t taosAddRef(int rsetId, void *p) rid = atomic_add_fetch_64(&pSet->rid, 1); hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode->p = p; pNode->rid = rid; @@ -185,26 +183,22 @@ int64_t taosAddRef(int rsetId, void *p) uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count); - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); return rid; } -int taosRemoveRef(int rsetId, int64_t rid) -{ - return taosDecRefCount(rsetId, rid, 1); -} +int32_t taosRemoveRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 1); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosAcquireRef(int rsetId, int64_t rid) -{ - int hash; +void *taosAcquireRef(int32_t rsetId, int64_t rid) { + int32_t hash; SRefNode *pNode; SRefSet *pSet; void *p = NULL; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - //uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + // uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } @@ -225,7 +219,7 @@ void *taosAcquireRef(int rsetId, int64_t rid) } hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; @@ -251,20 +245,17 @@ void *taosAcquireRef(int rsetId, int64_t rid) uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid); } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); taosDecRsetCount(pSet); return p; } -int taosReleaseRef(int rsetId, int64_t rid) -{ - return taosDecRefCount(rsetId, rid, 0); -} +int32_t taosReleaseRef(int32_t rsetId, int64_t rid) { return taosDecRefCount(rsetId, rid, 0); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid -void *taosIterateRef(int rsetId, int64_t rid) { +void *taosIterateRef(int32_t rsetId, int64_t rid) { SRefNode *pNode = NULL; SRefSet *pSet; @@ -292,10 +283,10 @@ void *taosIterateRef(int rsetId, int64_t rid) { do { newP = NULL; - int hash = 0; + int32_t hash = 0; if (rid > 0) { hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; while (pNode) { @@ -306,7 +297,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { if (pNode == NULL) { uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); taosDecRsetCount(pSet); return NULL; } @@ -319,14 +310,14 @@ void *taosIterateRef(int rsetId, int64_t rid) { pNode = pNode->next; } if (pNode == NULL) { - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); hash++; } } if (pNode == NULL) { for (; hash < pSet->max; ++hash) { - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; if (pNode) { // check first place @@ -336,14 +327,14 @@ void *taosIterateRef(int rsetId, int64_t rid) { } if (pNode) break; } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); } } if (pNode) { pNode->count++; // acquire it newP = pNode->p; - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); } else { uTrace("rsetId:%d the list is over", rsetId); @@ -358,22 +349,21 @@ void *taosIterateRef(int rsetId, int64_t rid) { return newP; } -int taosListRef() { +int32_t taosListRef() { SRefSet *pSet; SRefNode *pNode; - int num = 0; + int32_t num = 0; pthread_mutex_lock(&tsRefMutex); - for (int i = 0; i < TSDB_REF_OBJECTS; ++i) { + for (int32_t i = 0; i < TSDB_REF_OBJECTS; ++i) { pSet = tsRefSetList + i; - if (pSet->state == TSDB_REF_STATE_EMPTY) - continue; + if (pSet->state == TSDB_REF_STATE_EMPTY) continue; uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); - for (int j=0; j < pSet->max; ++j) { + for (int32_t j = 0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; while (pNode) { @@ -389,12 +379,12 @@ int taosListRef() { return num; } -static int taosDecRefCount(int rsetId, int64_t rid, int remove) { - int hash; +static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove) { + int32_t hash; SRefSet *pSet; SRefNode *pNode; - int released = 0; - int code = 0; + int32_t released = 0; + int32_t code = 0; if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid); @@ -416,12 +406,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); + taosLockList(pSet->lockedBy + hash); pNode = pSet->nodeList[hash]; while (pNode) { - if (pNode->rid == rid) - break; + if (pNode->rid == rid) break; pNode = pNode->next; } @@ -436,13 +425,13 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { } else { pSet->nodeList[hash] = pNode->next; } - + if (pNode->next) { pNode->next->prev = pNode->prev; } released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); @@ -450,10 +439,11 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { code = -1; } - taosUnlockList(pSet->lockedBy+hash); + taosUnlockList(pSet->lockedBy + hash); if (released) { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, + pNode); (*pSet->fp)(pNode->p); free(pNode); @@ -465,7 +455,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { static void taosLockList(int64_t *lockedBy) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_64(lockedBy, 0, tid) != 0) { if (++i % 100 == 0) { sched_yield(); @@ -480,9 +470,7 @@ static void taosUnlockList(int64_t *lockedBy) { } } -static void taosInitRefModule(void) { - pthread_mutex_init(&tsRefMutex, NULL); -} +static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); @@ -511,4 +499,3 @@ static void taosDecRsetCount(SRefSet *pSet) { pthread_mutex_unlock(&tsRefMutex); } - diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 915edc45ceb99d75a36bf31eb93f550ccc98d561..740e742bad5e9b9f9e50194bcc02aadc1146cba3 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -13,35 +13,35 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tdef.h" -#include "tutil.h" -#include "ulog.h" +#define _DEFAULT_SOURCE #include "tsched.h" +#include "tdef.h" +#include "tlog.h" #include "ttimer.h" +#include "tutil.h" -#define DUMP_SCHEDULER_TIME_WINDOW 30000 //every 30sec, take a snap shot of task queue. +#define DUMP_SCHEDULER_TIME_WINDOW 30000 // every 30sec, take a snap shot of task queue. typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; tsem_t fullSem; pthread_mutex_t queueMutex; - int fullSlot; - int emptySlot; - int queueSize; - int numOfThreads; - pthread_t * qthread; - SSchedMsg * queue; + int32_t fullSlot; + int32_t emptySlot; + int32_t queueSize; + int32_t numOfThreads; + pthread_t *qthread; + SSchedMsg *queue; bool stop; - void* pTmrCtrl; - void* pTimer; + void *pTmrCtrl; + void *pTimer; } SSchedQueue; static void *taosProcessSchedQueue(void *param); -static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); +static void taosDumpSchedulerStatus(void *qhandle, void *tmrId); -void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { +void *taosInitScheduler(int32_t queueSize, int32_t numOfThreads, const char *label) { SSchedQueue *pSched = (SSchedQueue *)calloc(sizeof(SSchedQueue), 1); if (pSched == NULL) { uError("%s: no enough memory for pSched", label); @@ -63,7 +63,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pSched->queueSize = queueSize; - tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow + tstrncpy(pSched->label, label, sizeof(pSched->label)); // fix buffer overflow pSched->fullSlot = 0; pSched->emptySlot = 0; @@ -74,7 +74,7 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { return NULL; } - if (tsem_init(&pSched->emptySem, 0, (unsigned int)pSched->queueSize) != 0) { + if (tsem_init(&pSched->emptySem, 0, (uint32_t)pSched->queueSize) != 0) { uError("init %s:empty semaphore failed(%s)", label, strerror(errno)); taosCleanUpScheduler(pSched); return NULL; @@ -87,11 +87,11 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { } pSched->stop = false; - for (int i = 0; i < numOfThreads; ++i) { + for (int32_t i = 0; i < numOfThreads; ++i) { pthread_attr_t attr; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - int code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); + int32_t code = pthread_create(pSched->qthread + i, &attr, taosProcessSchedQueue, (void *)pSched); pthread_attr_destroy(&attr); if (code != 0) { uError("%s: failed to create rpc thread(%s)", label, strerror(errno)); @@ -106,8 +106,8 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { return (void *)pSched; } -void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { - SSchedQueue* pSched = taosInitScheduler(queueSize, numOfThreads, label); +void *taosInitSchedulerWithInfo(int32_t queueSize, int32_t numOfThreads, const char *label, void *tmrCtrl) { + SSchedQueue *pSched = taosInitScheduler(queueSize, numOfThreads, label); if (tmrCtrl != NULL && pSched != NULL) { pSched->pTmrCtrl = tmrCtrl; @@ -120,7 +120,7 @@ void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *lab void *taosProcessSchedQueue(void *scheduler) { SSchedMsg msg; SSchedQueue *pSched = (SSchedQueue *)scheduler; - int ret = 0; + int32_t ret = 0; char name[16] = {0}; snprintf(name, tListLen(name), "%s-taskQ", pSched->label); @@ -165,7 +165,7 @@ void *taosProcessSchedQueue(void *scheduler) { void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { SSchedQueue *pSched = (SSchedQueue *)queueScheduler; - int ret = 0; + int32_t ret = 0; if (pSched == NULL) { uError("sched is not ready, msg:%p is dropped", pMsg); @@ -201,12 +201,12 @@ void taosCleanUpScheduler(void *param) { if (pSched == NULL) return; pSched->stop = true; - for (int i = 0; i < pSched->numOfThreads; ++i) { - if (taosCheckPthreadValid(pSched->qthread[i])) { + for (int32_t i = 0; i < pSched->numOfThreads; ++i) { + if (taosCheckPthreadValid(pSched->qthread[i])) { tsem_post(&pSched->fullSem); } } - for (int i = 0; i < pSched->numOfThreads; ++i) { + for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { pthread_join(pSched->qthread[i], NULL); } @@ -215,14 +215,14 @@ void taosCleanUpScheduler(void *param) { tsem_destroy(&pSched->emptySem); tsem_destroy(&pSched->fullSem); pthread_mutex_destroy(&pSched->queueMutex); - + if (pSched->pTimer) { taosTmrStopA(&pSched->pTimer); } if (pSched->queue) free(pSched->queue); if (pSched->qthread) free(pSched->qthread); - free(pSched); // fix memory leak + free(pSched); // fix memory leak } // for debug purpose, dump the scheduler status every 1min. @@ -231,11 +231,11 @@ void taosDumpSchedulerStatus(void *qhandle, void *tmrId) { if (pSched == NULL || pSched->pTimer == NULL || pSched->pTimer != tmrId) { return; } - + int32_t size = ((pSched->emptySlot - pSched->fullSlot) + pSched->queueSize) % pSched->queueSize; if (size > 0) { uDebug("scheduler:%s, current tasks in queue:%d, task thread:%d", pSched->label, size, pSched->numOfThreads); } - + taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); } diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 328d3da5a4a4e40c508da3881fb2eba48c939d02..6b89ed2c430f240a3b1a219df85b9b876f560c9b 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -14,15 +14,14 @@ * along with this program. If not, see . */ -#include "os.h" - -#include "compare.h" +#define _DEFAULT_SOURCE #include "tskiplist.h" +#include "tcompare.h" +#include "tlog.h" #include "tutil.h" -#include "ulog.h" -static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); +static int32_t initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -33,10 +32,9 @@ static SSkipListNode *tSkipListNewNode(uint8_t level); static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipListNode **direction, bool isForward, bool hasDup); - -static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); -static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); -static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList); +static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, @@ -140,21 +138,21 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; bool hasDup = false; - char * pKey = NULL; - char * pDataKey = NULL; - int compare = 0; + char *pKey = NULL; + char *pDataKey = NULL; + int32_t compare = 0; tSkipListWLock(pSkipList); - void* pData = iterate(iter); - if(pData == NULL) return; + void *pData = iterate(iter); + if (pData == NULL) return; // backward to put the first data hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); tSkipListPutImpl(pSkipList, pData, backward, false, hasDup); - for (int level = 0; level < pSkipList->maxLevel; level++) { + for (int32_t level = 0; level < pSkipList->maxLevel; level++) { forward[level] = SL_NODE_GET_BACKWARD_POINTER(backward[level], level); } @@ -167,12 +165,12 @@ void tSkipListPutBatchByIter(SSkipList *pSkipList, void *iter, iter_next_fn_t it pKey = SL_GET_MAX_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare > 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); } } else { SSkipListNode *px = pSkipList->pHead; - for (int i = pSkipList->maxLevel - 1; i >= 0; --i) { + for (int32_t i = pSkipList->maxLevel - 1; i >= 0; --i) { if (i < pSkipList->level) { // set new px if (forward[i] != pSkipList->pHead) { @@ -359,7 +357,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; - char * prev = NULL; + char *prev = NULL; while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); @@ -435,21 +433,21 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t return iter; } -static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListWLock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_wrlock(pSkipList->lock); } return 0; } -static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListRLock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_rdlock(pSkipList->lock); } return 0; } -static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { +static FORCE_INLINE int32_t tSkipListUnlock(SSkipList *pSkipList) { if (pSkipList->lock) { return pthread_rwlock_unlock(pSkipList->lock); } @@ -457,12 +455,12 @@ static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { } static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { - int compare = 0; + int32_t compare = 0; bool hasDupKey = false; - char * pDataKey = pSkipList->keyFn(pData); + char *pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = pSkipList->pTail; } } else { @@ -472,7 +470,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, pKey = SL_GET_MAX_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare >= 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = pSkipList->pTail; } @@ -483,7 +481,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, pKey = SL_GET_MIN_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); if (compare < 0) { - for (int i = 0; i < pSkipList->maxLevel; i++) { + for (int32_t i = 0; i < pSkipList->maxLevel; i++) { backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i); } @@ -491,7 +489,7 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, } SSkipListNode *px = pSkipList->pTail; - for (int i = pSkipList->maxLevel - 1; i >= 0; --i) { + for (int32_t i = pSkipList->maxLevel - 1; i >= 0; --i) { if (i < pSkipList->level) { SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i); while (p != pSkipList->pHead) { @@ -534,7 +532,8 @@ static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) // Function must be called after calling tSkipListRemoveNodeImpl() function static void tSkipListCorrectLevel(SSkipList *pSkipList) { - while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + while (pSkipList->level > 0 && + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { pSkipList->level -= 1; } } @@ -638,7 +637,7 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ return pNode; } -static int initForwardBackwardPtr(SSkipList *pSkipList) { +static int32_t initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; // head info @@ -687,12 +686,12 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL pSkipList->insertHandleFn->args[1] = pNode->pData; pData = genericInvoke(pSkipList->insertHandleFn); } - if(pData) { + if (pData) { atomic_store_ptr(&(pNode->pData), pData); } } else { - //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! - if(pSkipList->insertHandleFn) { + // for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! + if (pSkipList->insertHandleFn) { pSkipList->insertHandleFn->args[0] = NULL; pSkipList->insertHandleFn->args[1] = NULL; genericInvoke(pSkipList->insertHandleFn); diff --git a/source/util/src/tstep.c b/source/util/src/tstep.c deleted file mode 100644 index d840b119fb8358ef581b2bb1a8154dbed992dfc9..0000000000000000000000000000000000000000 --- a/source/util/src/tstep.c +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "taoserror.h" -#include "tstep.h" - -typedef struct { - char * name; - InitFp initFp; - CleanupFp cleanupFp; -} SStep; - -typedef struct SSteps { - int32_t cursize; - int32_t maxsize; - SStep * steps; - ReportFp reportFp; -} SSteps; - -SSteps *taosStepInit(int32_t maxsize, ReportFp fp) { - SSteps *steps = calloc(1, sizeof(SSteps)); - if (steps == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - steps->maxsize = maxsize; - steps->cursize = 0; - steps->steps = calloc(maxsize, sizeof(SStep)); - steps->reportFp = fp; - - return steps; -} - -int32_t taosStepAdd(struct SSteps *steps, char *name, InitFp initFp, CleanupFp cleanupFp) { - if (steps == NULL) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - if (steps->cursize >= steps->maxsize) { - uError("failed to add step since up to the maxsize"); - terrno = TSDB_CODE_OUT_OF_RANGE; - return -1; - } - - SStep step = {.name = name, .initFp = initFp, .cleanupFp = cleanupFp}; - steps->steps[steps->cursize++] = step; - return 0; -} - -static void taosStepCleanupImp(SSteps *steps, int32_t pos) { - for (int32_t s = pos; s >= 0; s--) { - SStep *step = steps->steps + s; - uDebug("step:%s will cleanup", step->name); - if (step->cleanupFp != NULL) { - (*step->cleanupFp)(); - } - } -} - -int32_t taosStepExec(SSteps *steps) { - if (steps == NULL) { - terrno = TSDB_CODE_INVALID_PTR; - return -1; - } - - for (int32_t s = 0; s < steps->cursize; s++) { - SStep *step = steps->steps + s; - if (step->initFp == NULL) continue; - - if (steps->reportFp != NULL) { - (*steps->reportFp)(step->name, "start initialize"); - } - - int32_t code = (*step->initFp)(); - if (code != 0) { - uDebug("step:%s will cleanup", step->name); - taosStepCleanupImp(steps, s); - return code; - } - - uInfo("step:%s is initialized", step->name); - - if (steps->reportFp != NULL) { - (*steps->reportFp)(step->name, "initialize completed"); - } - } - - return 0; -} - -void taosStepCleanup(SSteps *steps) { - if (steps == NULL) return; - taosStepCleanupImp(steps, steps->cursize - 1); -} \ No newline at end of file diff --git a/source/util/src/tstrbuild.c b/source/util/src/tstrbuild.c index 230bff42f51abe19f6b8bb7b866db45f3475832c..f191f69986a840c520703979f503406ab94ba9bc 100644 --- a/source/util/src/tstrbuild.c +++ b/source/util/src/tstrbuild.c @@ -12,7 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" + +#define _DEFAULT_SOURCE #include "tstrbuild.h" void taosStringBuilderEnsureCapacity(SStringBuilder* sb, size_t size) { diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 44fce1c882c4f0d7d66f36d378db1594e59f19ea..f9e28d7b6256bb61a51f8c76bc85da338c4f9d60 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -13,14 +13,9 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tthread.h" -#include "os.h" -#include "taoserror.h" -#include "tdef.h" -#include "tutil.h" -#include "ulog.h" -// create new thread pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_attr_t thattr; @@ -36,7 +31,6 @@ pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { return pthread; } -// destory thread bool taosDestoryThread(pthread_t* pthread) { if (pthread == NULL) return false; if (taosThreadRunning(pthread)) { @@ -48,10 +42,9 @@ bool taosDestoryThread(pthread_t* pthread) { return true; } -// thread running return true bool taosThreadRunning(pthread_t* pthread) { if (pthread == NULL) return false; - int ret = pthread_kill(*pthread, 0); + int32_t ret = pthread_kill(*pthread, 0); if (ret == ESRCH) return false; if (ret == EINVAL) return false; // alive diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 65101a5e07577e29c4678bec5dba1ac3a38a7347..abb42ef28d6d0135cce72ac494e21b5b85d51418 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -13,12 +13,11 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "ttimer.h" -#include "os.h" #include "taoserror.h" #include "tlog.h" #include "tsched.h" -#include "tutil.h" #define tmrFatal(...) \ { \ @@ -57,9 +56,9 @@ } \ } -#define TIMER_STATE_WAITING 0 -#define TIMER_STATE_EXPIRED 1 -#define TIMER_STATE_STOPPED 2 +#define TIMER_STATE_WAITING 0 +#define TIMER_STATE_EXPIRED 1 +#define TIMER_STATE_STOPPED 2 #define TIMER_STATE_CANCELED 3 typedef union _tmr_ctrl_t { @@ -111,17 +110,16 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tmrDebugFlag = 131; -uint32_t tsMaxTmrCtrl = 512; +int32_t tsMaxTmrCtrl = 512; static pthread_once_t tmrModuleInit = PTHREAD_ONCE_INIT; static pthread_mutex_t tmrCtrlMutex; static tmr_ctrl_t* tmrCtrls; static tmr_ctrl_t* unusedTmrCtrl = NULL; static void* tmrQhandle; -static int numOfTmrCtrl = 0; +static int32_t numOfTmrCtrl = 0; -int taosTmrThreads = 1; +int32_t taosTmrThreads = 1; static uintptr_t nextTimerId = 0; static time_wheel_t wheels[] = { @@ -149,7 +147,7 @@ static void timerDecRef(tmr_obj_t* timer) { static void lockTimerList(timer_list_t* list) { int64_t tid = taosGetSelfPthreadId(); - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_64(&(list->lockedBy), 0, tid) != 0) { if (++i % 1000 == 0) { sched_yield(); @@ -323,7 +321,7 @@ static void addToExpired(tmr_obj_t* head) { } } -static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int mseconds, void* param, tmr_ctrl_t* ctrl) { +static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, tmr_ctrl_t* ctrl) { uintptr_t id = getNextTimerId(); timer->id = id; timer->state = TIMER_STATE_WAITING; @@ -347,7 +345,7 @@ static uintptr_t doStartTimer(tmr_obj_t* timer, TAOS_TMR_CALLBACK fp, int msecon return id; } -tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle) { +tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; if (ctrl == NULL || ctrl->label[0] == 0) { return NULL; @@ -362,10 +360,10 @@ tmr_h taosTmrStart(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle return (tmr_h)doStartTimer(timer, fp, mseconds, param, ctrl); } -static void taosTimerLoopFunc(int signo) { +static void taosTimerLoopFunc(int32_t signo) { int64_t now = taosGetMonotonicMs(); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { // `expried` is a temporary expire list. // expired timers are first add to this list, then move // to expired queue as a batch to improve performance. @@ -472,7 +470,7 @@ bool taosTmrStopA(tmr_h* timerId) { return ret; } -bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, tmr_h* pTmrId) { +bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void* param, void* handle, tmr_h* pTmrId) { tmr_ctrl_t* ctrl = (tmr_ctrl_t*)handle; if (ctrl == NULL || ctrl->label[0] == 0) { return false; @@ -501,7 +499,7 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int mseconds, void* param, void* handle, // wait until there's no other reference to this timer, // so that we can reuse this timer safely. - for (int i = 1; atomic_load_8(&timer->refCount) > 1; ++i) { + for (int32_t i = 1; atomic_load_8(&timer->refCount) > 1; ++i) { if (i % 1000 == 0) { sched_yield(); } @@ -533,7 +531,7 @@ static void taosTmrModuleInit(void) { pthread_mutex_init(&tmrCtrlMutex, NULL); int64_t now = taosGetMonotonicMs(); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; if (pthread_mutex_init(&wheel->mutex, NULL) != 0) { tmrError("failed to create the mutex for wheel, reason:%s", strerror(errno)); @@ -562,7 +560,7 @@ static void taosTmrModuleInit(void) { tmrDebug("timer module is initialized, number of threads: %d", taosTmrThreads); } -void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) { +void* taosTmrInit(int32_t maxNumOfTmrs, int32_t resolution, int32_t longest, const char* label) { const char* ret = taosMonotonicInit(); tmrDebug("ttimer monotonic clock source:%s", ret); @@ -608,7 +606,7 @@ void taosTmrCleanUp(void* handle) { taosCleanUpScheduler(tmrQhandle); - for (int i = 0; i < tListLen(wheels); i++) { + for (int32_t i = 0; i < tListLen(wheels); i++) { time_wheel_t* wheel = wheels + i; pthread_mutex_destroy(&wheel->mutex); free(wheel->slots); diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index 58a2e57f7c527cf8e75e8e0d3acf5beab0c4e6d7..7bd671a56ca42f9be79e123af734f1ba19ea83d6 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tdef.h" +#define _DEFAULT_SOURCE +#include "tutil.h" int32_t strdequote(char *z) { if (z == NULL) { @@ -47,38 +47,38 @@ int32_t strdequote(char *z) { return j + 1; // only one quote, do nothing } -int32_t strRmquote(char *z, int32_t len){ - // delete escape character: \\, \', \" - char delim = z[0]; - if (delim != '\'' && delim != '\"') { - return len; - } - - int32_t cnt = 0; - int32_t j = 0; - for (uint32_t k = 1; k < len - 1; ++k) { - if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { - if (z[k] == '\\' && z[k + 1] == '_') { - //match '_' self - } else { - z[j] = z[k + 1]; - cnt++; - j++; - k++; - continue; - } +int32_t strRmquote(char *z, int32_t len) { + // delete escape character: \\, \', \" + char delim = z[0]; + if (delim != '\'' && delim != '\"') { + return len; + } + + int32_t cnt = 0; + int32_t j = 0; + for (uint32_t k = 1; k < len - 1; ++k) { + if (z[k] == '\\' || (z[k] == delim && z[k + 1] == delim)) { + if (z[k] == '\\' && z[k + 1] == '_') { + // match '_' self + } else { + z[j] = z[k + 1]; + cnt++; + j++; + k++; + continue; } - - z[j] = z[k]; - j++; } - - z[j] = 0; - - return len - 2 - cnt; + + z[j] = z[k]; + j++; + } + + z[j] = 0; + + return len - 2 - cnt; } -int32_t strndequote(char *dst, const char* z, int32_t len) { +int32_t strndequote(char *dst, const char *z, int32_t len) { assert(dst != NULL); if (z == NULL || len == 0) { return 0; @@ -90,7 +90,7 @@ int32_t strndequote(char *dst, const char* z, int32_t len) { while (z[i] != 0) { if (z[i] == quote) { if (z[i + 1] == quote) { - dst[j++] = (char) quote; + dst[j++] = (char)quote; i++; } else { dst[j++] = 0; @@ -139,7 +139,7 @@ size_t strtrim(char *z) { } else if (j != i) { z[i] = 0; } - + return i; } @@ -168,11 +168,11 @@ char **strsplit(char *z, const char *delim, int32_t *num) { char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { for (int32_t i = 0; i < len; ++i) { - // skip the needle in quote, jump to the end of quoted string if (skipquote && (haystack[i] == '\'' || haystack[i] == '"')) { char quote = haystack[i++]; - while(i < len && haystack[i++] != quote); + while (i < len && haystack[i++] != quote) + ; if (i >= len) { return NULL; } @@ -186,9 +186,9 @@ char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote) { return NULL; } -char* strtolower(char *dst, const char *src) { +char *strtolower(char *dst, const char *src) { int32_t esc = 0; - char quote = 0, *p = dst, c; + char quote = 0, *p = dst, c; assert(dst != NULL); @@ -213,9 +213,9 @@ char* strtolower(char *dst, const char *src) { return dst; } -char* strntolower(char *dst, const char *src, int32_t n) { +char *strntolower(char *dst, const char *src, int32_t n) { int32_t esc = 0; - char quote = 0, *p = dst, c; + char quote = 0, *p = dst, c; assert(dst != NULL); if (n == 0) { @@ -243,7 +243,7 @@ char* strntolower(char *dst, const char *src, int32_t n) { return dst; } -char* strntolower_s(char *dst, const char *src, int32_t n) { +char *strntolower_s(char *dst, const char *src, int32_t n) { char *p = dst, c; assert(dst != NULL); @@ -265,7 +265,7 @@ char* strntolower_s(char *dst, const char *src, int32_t n) { char *paGetToken(char *string, char **token, int32_t *tokenLen) { char quote = 0; - + while (*string != 0) { if (*string == ' ' || *string == '\t') { ++string; @@ -346,8 +346,8 @@ char *strbetween(char *string, char *begin, char *end) { char *result = NULL; char *_begin = strstr(string, begin); if (_begin != NULL) { - char *_end = strstr(_begin + strlen(begin), end); - int32_t size = (int32_t)(_end - _begin); + char *_end = strstr(_begin + strlen(begin), end); + int32_t size = (int32_t)(_end - _begin); if (_end != NULL && size > 0) { result = (char *)calloc(1, size); memcpy(result, _begin + strlen(begin), size - +strlen(begin)); @@ -401,11 +401,12 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { } char *taosIpStr(uint32_t ipInt) { - static char ipStrArray[3][30]; + static char ipStrArray[3][30]; static int32_t ipStrIndex = 0; char *ipStr = ipStrArray[(ipStrIndex++) % 3]; - //sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); + // sprintf(ipStr, "0x%x:%u.%u.%u.%u", ipInt, ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt + // >> 24)); sprintf(ipStr, "%u.%u.%u.%u", ipInt & 0xFF, (ipInt >> 8) & 0xFF, (ipInt >> 16) & 0xFF, (uint8_t)(ipInt >> 24)); return ipStr; } diff --git a/source/util/src/tversion.c b/source/util/src/tversion.c index 3944bd51327a8516dcf87e65e8fa1b0ae74fe90a..c70fdc87a6363b1a9fd08bddef3e1b3d0439162e 100644 --- a/source/util/src/tversion.c +++ b/source/util/src/tversion.c @@ -14,10 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "os.h" +#include "tversion.h" #include "taoserror.h" -#include "tdef.h" -#include "ulog.h" int32_t taosVersionStrToInt(const char *vstr, int32_t *vint) { if (vstr == NULL) { diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index ca4e5a6f30672b79162056e73f338983ba41596d..1657a85ee8c85ffc70e088ba307cdb37d4873417 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "tworker.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef void *(*ThreadFp)(void *param); @@ -71,8 +71,8 @@ static void *tQWorkerThreadFp(SQWorker *worker) { SQWorkerPool *pool = worker->pool; FItem fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -151,8 +151,8 @@ static void *tFWorkerThreadFp(SQWorker *worker) { SQWorkerPool *pool = worker->pool; FItem fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -240,8 +240,8 @@ static void *tWWorkerThreadFp(SWWorker *worker) { SWWorkerPool *pool = worker->pool; FItems fp = NULL; - void * msg = NULL; - void * ahandle = NULL; + void *msg = NULL; + void *ahandle = NULL; int32_t numOfMsgs = 0; int32_t qtype = 0; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index ee0ade03b858707b5ef1bafb2554798709271e98..d6b779b6e32b98b0f729d71a428bd4ea5512f731 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -51,4 +51,12 @@ target_link_libraries(queue_test os util gtest_main) add_test( NAME queue_test COMMAND queue_test +) + +# cfgTest +add_executable(cfgTest "cfgTest.cpp") +target_link_libraries(cfgTest os util gtest_main) +add_test( + NAME cfgTest + COMMAND cfgTest ) \ No newline at end of file diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..712fb2d09a240037e99fd144a5030bb670ef606a --- /dev/null +++ b/source/util/test/cfgTest.cpp @@ -0,0 +1,131 @@ +/** + * @file cfgTest.cpp + * @author slguan (slguan@taosdata.com) + * @brief config module tests + * @version 1.0 + * @date 2022-02-20 + * + * @copyright Copyright (c) 2022 + * + */ + +#include +#include "tconfig.h" + +class CfgTest : public ::testing::Test { + protected: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} + + public: + void SetUp() override {} + void TearDown() override {} + void InitCfg(SConfig *pConfig); + + static const char *pConfig; +}; + +const char *CfgTest::pConfig; + +TEST_F(CfgTest, 01_Str) { + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_DEFAULT), "default"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); + EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); + + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_NONE), "none"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_BOOL), "bool"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT32), "int32"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_INT64), "int64"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_FLOAT), "float"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_STRING), "string"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(CFG_DTYPE_DIR), "dir"); + EXPECT_STREQ(cfgDtypeStr(ECfgDataType(1024)), "invalid"); +} + +TEST_F(CfgTest, 02_Basic) { + SConfig *pConfig = cfgInit(); + ASSERT_NE(pConfig, nullptr); + + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", "/tmp", 0), 0); + + EXPECT_EQ(cfgGetSize(pConfig), 6); + + int32_t size = taosArrayGetSize(pConfig->array); + for (int32_t i = 0; i < size; ++i) { + SConfigItem *pItem = (SConfigItem *)taosArrayGet(pConfig->array, i); + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); + break; + case CFG_DTYPE_INT32: + printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->i32); + break; + case CFG_DTYPE_INT64: + printf("index:%d, cfg:%s value:%" PRId64 "\n", size, pItem->name, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + printf("index:%d, cfg:%s value:%f\n", size, pItem->name, pItem->fval); + break; + case CFG_DTYPE_STRING: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + case CFG_DTYPE_DIR: + printf("index:%d, cfg:%s value:%s\n", size, pItem->name, pItem->str); + break; + default: + printf("index:%d, cfg:%s invalid cfg dtype:%d\n", size, pItem->name, pItem->dtype); + break; + } + } + EXPECT_EQ(cfgGetSize(pConfig), 6); + + SConfigItem *pItem = cfgGetItem(pConfig, "test_bool"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_BOOL); + EXPECT_STREQ(pItem->name, "test_bool"); + EXPECT_EQ(pItem->bval, 0); + + pItem = cfgGetItem(pConfig, "test_int32"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT32); + EXPECT_STREQ(pItem->name, "test_int32"); + EXPECT_EQ(pItem->i32, 1); + + pItem = cfgGetItem(pConfig, "test_int64"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_INT64); + EXPECT_STREQ(pItem->name, "test_int64"); + EXPECT_EQ(pItem->i64, 2); + + pItem = cfgGetItem(pConfig, "test_float"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_FLOAT); + EXPECT_STREQ(pItem->name, "test_float"); + EXPECT_EQ(pItem->fval, 3); + + pItem = cfgGetItem(pConfig, "test_string"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_STRING); + EXPECT_STREQ(pItem->name, "test_string"); + EXPECT_STREQ(pItem->str, "4"); + + pItem = cfgGetItem(pConfig, "test_dir"); + EXPECT_EQ(pItem->stype, CFG_STYPE_DEFAULT); + EXPECT_EQ(pItem->dtype, CFG_DTYPE_DIR); + EXPECT_STREQ(pItem->name, "test_dir"); + EXPECT_STREQ(pItem->str, "/tmp"); + + cfgCleanup(pConfig); +} diff --git a/source/util/test/encodeTest.cpp b/source/util/test/encodeTest.cpp index 1b13c102a5a14103f51ceaf337d346723fd6f6f0..5505a6207f676d7f34d3476ca83cb33d24d76d9d 100644 --- a/source/util/test/encodeTest.cpp +++ b/source/util/test/encodeTest.cpp @@ -2,7 +2,7 @@ #include "gtest/gtest.h" -#include "encode.h" +#include "tencode.h" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshift-count-overflow" @@ -12,41 +12,41 @@ #define BUF_SIZE 64 td_endian_t endian_arr[2] = {TD_LITTLE_ENDIAN, TD_BIG_ENDIAN}; -static int encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } -static int encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } -static int encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); } -static int encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); } -static int encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); } -static int encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); } -static int encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); } -static int encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); } - -static int decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } -static int decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } -static int decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); } -static int decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); } -static int decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); } -static int decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); } -static int decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); } -static int decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); } - -static int encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } -static int encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } -static int encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); } -static int encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); } -static int encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); } -static int encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); } -static int encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); } -static int encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); } - -static int decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } -static int decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } -static int decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); } -static int decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); } -static int decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); } -static int decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); } -static int decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); } -static int decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); } +static int32_t encode(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int32_t encode(SCoder *pCoder, int16_t val) { return tEncodeI16(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint16_t val) { return tEncodeU16(pCoder, val); } +static int32_t encode(SCoder *pCoder, int32_t val) { return tEncodeI32(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint32_t val) { return tEncodeU32(pCoder, val); } +static int32_t encode(SCoder *pCoder, int64_t val) { return tEncodeI64(pCoder, val); } +static int32_t encode(SCoder *pCoder, uint64_t val) { return tEncodeU64(pCoder, val); } + +static int32_t decode(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int32_t decode(SCoder *pCoder, int16_t *val) { return tDecodeI16(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint16_t *val) { return tDecodeU16(pCoder, val); } +static int32_t decode(SCoder *pCoder, int32_t *val) { return tDecodeI32(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint32_t *val) { return tDecodeU32(pCoder, val); } +static int32_t decode(SCoder *pCoder, int64_t *val) { return tDecodeI64(pCoder, val); } +static int32_t decode(SCoder *pCoder, uint64_t *val) { return tDecodeU64(pCoder, val); } + +static int32_t encodev(SCoder *pCoder, int8_t val) { return tEncodeI8(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint8_t val) { return tEncodeU8(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int16_t val) { return tEncodeI16v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint16_t val) { return tEncodeU16v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int32_t val) { return tEncodeI32v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint32_t val) { return tEncodeU32v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, int64_t val) { return tEncodeI64v(pCoder, val); } +static int32_t encodev(SCoder *pCoder, uint64_t val) { return tEncodeU64v(pCoder, val); } + +static int32_t decodev(SCoder *pCoder, int8_t *val) { return tDecodeI8(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint8_t *val) { return tDecodeU8(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int16_t *val) { return tDecodeI16v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint16_t *val) { return tDecodeU16v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int32_t *val) { return tDecodeI32v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint32_t *val) { return tDecodeU32v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, int64_t *val) { return tDecodeI64v(pCoder, val); } +static int32_t decodev(SCoder *pCoder, uint64_t *val) { return tDecodeU64v(pCoder, val); } template static void simple_encode_decode_func(bool var_len) { @@ -211,7 +211,7 @@ typedef struct { char * A_c; } SStructA_v1; -static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { +static int32_t tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI32(pCoder, pSAV1->A_a) < 0) return -1; @@ -222,7 +222,7 @@ static int tSStructA_v1_encode(SCoder *pCoder, const SStructA_v1 *pSAV1) { return 0; } -static int tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { +static int32_t tSStructA_v1_decode(SCoder *pCoder, SStructA_v1 *pSAV1) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32(pCoder, &pSAV1->A_a) < 0) return -1; @@ -246,7 +246,7 @@ typedef struct { int16_t A_e; } SStructA_v2; -static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { +static int32_t tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeI32(pCoder, pSAV2->A_a) < 0) return -1; @@ -261,7 +261,7 @@ static int tSStructA_v2_encode(SCoder *pCoder, const SStructA_v2 *pSAV2) { return 0; } -static int tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { +static int32_t tSStructA_v2_decode(SCoder *pCoder, SStructA_v2 *pSAV2) { if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32(pCoder, &pSAV2->A_a) < 0) return -1; @@ -291,7 +291,7 @@ typedef struct { int8_t v_b; } SFinalReq_v1; -static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { +static int32_t tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { if (tStartEncode(pCoder) < 0) return -1; if (tSStructA_v1_encode(pCoder, ps1->pA) < 0) return -1; @@ -302,7 +302,7 @@ static int tSFinalReq_v1_encode(SCoder *pCoder, const SFinalReq_v1 *ps1) { return 0; } -static int tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { +static int32_t tSFinalReq_v1_decode(SCoder *pCoder, SFinalReq_v1 *ps1) { if (tStartDecode(pCoder) < 0) return -1; ps1->pA = (SStructA_v1 *)TCODER_MALLOC(sizeof(*(ps1->pA)), pCoder); @@ -322,7 +322,7 @@ typedef struct { int16_t v_c; } SFinalReq_v2; -static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { +static int32_t tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { if (tStartEncode(pCoder) < 0) return -1; if (tSStructA_v2_encode(pCoder, ps2->pA) < 0) return -1; @@ -336,7 +336,7 @@ static int tSFinalReq_v2_encode(SCoder *pCoder, const SFinalReq_v2 *ps2) { return 0; } -static int tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { +static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) { if (tStartDecode(pCoder) < 0) return -1; ps2->pA = (SStructA_v2 *)TCODER_MALLOC(sizeof(*(ps2->pA)), pCoder); diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 7a4e8be5b7f908fdccfe524c2ec2729a6090ec90..cf11d6b5bf461a19ce4a6ee231ee92d0c5fbdb8b 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -1,6 +1,6 @@ #include "gtest/gtest.h" -#include "freelist.h" +#include "tfreelist.h" TEST(TD_UTIL_FREELIST_TEST, simple_test) { SFreeList fl; diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 8fa82162234dd60c9562a0ad5103f818b26260c9..4b9d8c5f515f397d81d299d554278c132c03d9b2 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -13,144 +13,144 @@ namespace { // simple test void simpleTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4096, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/"); int32_t pageId = 0; int32_t groupId = 0; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1); releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); - SFilePage* t = getBufPage(pResultBuf, pageId); + SFilePage* t = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t == pBufPage1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage4); - SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage5); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } void writeDownTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); *(int32_t*)(pBufPage->data) = nx; writePageId = pageId; releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseBufPage(pResultBuf, t4); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 5); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } void recyclePageTest() { SDiskbasedBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); int32_t pageId = 0; int32_t writePageId = 0; int32_t groupId = 0; int32_t nx = 12345; - SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* pBufPage = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); ASSERT_TRUE(pBufPage != NULL); releaseBufPage(pResultBuf, pBufPage); - SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t1 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage1 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t1 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); - SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t2 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage2 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t2 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); - SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t3 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage3 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t3 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); - SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t4 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage4 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t4 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseBufPage(pResultBuf, t4); - SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - SFilePage* t5 = getBufPage(pResultBuf, pageId); + SFilePage* pBufPage5 = static_cast(getNewBufPage(pResultBuf, groupId, &pageId)); + SFilePage* t5 = static_cast(getBufPage(pResultBuf, pageId)); ASSERT_TRUE(t5 == pBufPage5); ASSERT_TRUE(pageId == 5); // flush the written page to disk, and read it out again - SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + SFilePage* pBufPagex = static_cast(getBufPage(pResultBuf, writePageId)); *(int32_t*)(pBufPagex->data) = nx; writePageId = pageId; // update the data releaseBufPage(pResultBuf, pBufPagex); - SFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); + SFilePage* pBufPagex1 = static_cast(getBufPage(pResultBuf, 1)); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 6); - destroyResultBuf(pResultBuf); + destroyDiskbasedBuf(pResultBuf); } } // namespace diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 54aa6a5f346b481e387593fcbd40077ca1a979ec..586151d7827066cfa4fa840cbb862a03bb435989 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -8,7 +8,7 @@ #include "tlog.h" #include "tglobal.h" #include "taoserror.h" -#include "ulog.h" +#include "tlog.h" typedef struct { int refNum; @@ -158,7 +158,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 5000000, 10); + taosInitLog("tref.log", 10); SRefSpace *pSpaceList = (SRefSpace *) calloc(sizeof(SRefSpace), threads); pthread_t *pThreadList = (pthread_t *) calloc(sizeof(pthread_t), threads); diff --git a/tests b/tests index 4da4bb00c5615f0b1bea04340abef5d9d15a96b4..904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c 160000 --- a/tests +++ b/tests @@ -1 +1 @@ -Subproject commit 4da4bb00c5615f0b1bea04340abef5d9d15a96b4 +Subproject commit 904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c index 7fc8b1409a7602df48108d0e7f4763da48ed6497..33d25b67464e52cb6808f6b845963cf65ee49aac 100644 --- a/tools/shell/src/backup/shellCheck.c +++ b/tools/shell/src/backup/shellCheck.c @@ -116,7 +116,7 @@ static void *shellCheckThreadFp(void *arg) { char file[32] = {0}; snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - FILE *fp = fopen(file, "w"); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (!fp) { fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); return NULL; @@ -133,7 +133,7 @@ static void *shellCheckThreadFp(void *arg) { int32_t code = taos_errno(pSql); if (code != 0) { int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - fwrite(sql, 1, len, fp); + taosWriteFile(pFile, sql, len); atomic_add_fetch_32(&errorNum, 1); } @@ -145,8 +145,8 @@ static void *shellCheckThreadFp(void *arg) { taos_free_result(pSql); } - taosFsync(fileno(fp)); - fclose(fp); + taosFsync(pFile); + taosCloseFile(&pFile); return NULL; } diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c index a1413be1ce4ce6f67516fc09121115f30bbc56f0..f2ab468574e29d178cf15cf47c618b11cd869030 100644 --- a/tools/shell/src/backup/shellDarwin.c +++ b/tools/shell/src/backup/shellDarwin.c @@ -19,7 +19,7 @@ #include "shell.h" #include "shellCommand.h" -#include "tkey.h" +#include "tbase64.h" #include "tscLog.h" diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c index 222d69e854933095ec0aadaa8a67bf1c19954c3b..c0ab7ef461d3dcfa9df3b867153d255dd490c7ad 100644 --- a/tools/shell/src/backup/shellImport.c +++ b/tools/shell/src/backup/shellImport.c @@ -138,7 +138,6 @@ static void shellSourceFile(TAOS *con, char *fptr) { char * cmd = malloc(tsMaxSQLStringLen); size_t cmd_len = 0; char * line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -171,8 +170,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); @@ -182,7 +182,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { fprintf(stdout, "begin import file:%s\n", fname); int lineNo = 0; - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { ++lineNo; if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; @@ -215,9 +215,9 @@ static void shellSourceFile(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void* shellImportThreadFp(void *arg) diff --git a/tools/shell/src/backup/tnettest.c b/tools/shell/src/backup/tnettest.c index be8714387f3e693dcdaf673e5dec959a39a18912..772d92d8c6956a3eed1d5273df984c15b73cffc8 100644 --- a/tools/shell/src/backup/tnettest.c +++ b/tools/shell/src/backup/tnettest.c @@ -18,7 +18,7 @@ #include "taosdef.h" #include "tmsg.h" #include "taoserror.h" -#include "tulog.h" +#include "tlog.h" #include "tglobal.h" #include "tsocket.h" #include "trpc.h" diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4186f0dae1038f2336565a99b03953b0e99a1e8d..5a989937d8cfd41050bca0a077947a6feda9514b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -459,44 +459,44 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(FILE *fp, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { if (val == NULL) { - fprintf(fp, "%s", TSDB_DATA_NULL_STR); + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; } char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: - fprintf(fp, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char *)val))) == 1) ? 1 : 0)); break; case TSDB_DATA_TYPE_TINYINT: - fprintf(fp, "%d", *((int8_t *)val)); + taosFprintfFile(pFile, "%d", *((int8_t *)val)); break; case TSDB_DATA_TYPE_SMALLINT: - fprintf(fp, "%d", *((int16_t *)val)); + taosFprintfFile(pFile, "%d", *((int16_t *)val)); break; case TSDB_DATA_TYPE_INT: - fprintf(fp, "%d", *((int32_t *)val)); + taosFprintfFile(pFile, "%d", *((int32_t *)val)); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%" PRId64, *((int64_t *)val)); + taosFprintfFile(pFile, "%" PRId64, *((int64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - fprintf(fp, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - fprintf(fp, "%.9f", GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%.9f", GET_DOUBLE_VAL(val)); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: memcpy(buf, val, length); buf[length] = 0; - fprintf(fp, "\'%s\'", buf); + taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: formatTimestamp(buf, *(int64_t *)val, precision); - fprintf(fp, "'%s'", buf); + taosFprintfFile(pFile, "'%s'", buf); break; default: break; @@ -516,8 +516,9 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { return -1; } - FILE *fp = fopen(full_path.we_wordv[0], "w"); - if (fp == NULL) { + // FILE *fp = fopen(full_path.we_wordv[0], "w"); + TdFilePtr pFile = taosOpenFile(full_path.we_wordv[0], TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); wordfree(&full_path); return -1; @@ -531,29 +532,29 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { for (int col = 0; col < num_fields; col++) { if (col > 0) { - fprintf(fp, ","); + taosFprintfFile(pFile, ","); } - fprintf(fp, "%s", fields[col].name); + taosFprintfFile(pFile, "%s", fields[col].name); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); int numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); for (int i = 0; i < num_fields; i++) { if (i > 0) { - fputc(',', fp); + taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(fp, (const char *)row[i], fields + i, length[i], precision); + dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } - fputc('\n', fp); + taosFprintfFile(pFile, "\n"); numOfRows++; row = taos_fetch_row(tres); } while (row != NULL); result = 0; - fclose(fp); + taosCloseFile(&pFile); return numOfRows; } @@ -897,14 +898,14 @@ void read_history() { history.hstart = 0; history.hend = 0; char *line = NULL; - size_t line_size = 0; int read_size = 0; char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "r"); - if (f == NULL) { + // FILE *f = fopen(f_history, "r"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { #ifndef WINDOWS if (errno != ENOENT) { fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); @@ -913,7 +914,7 @@ void read_history() { return; } - while ((read_size = tgetline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -924,16 +925,17 @@ void read_history() { } } - free(line); - fclose(f); + if(line != NULL) free(line); + taosCloseFile(&pFile); } void write_history() { char f_history[TSDB_FILENAME_LEN]; get_history_path(f_history); - FILE *f = fopen(f_history, "w"); - if (f == NULL) { + // FILE *f = fopen(f_history, "w"); + TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pFile == NULL) { #ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); #endif @@ -942,12 +944,12 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { - fprintf(f, "%s\n", history.hist[i]); + taosFprintfFile(pFile, "%s\n", history.hist[i]); tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } - fclose(f); + taosCloseFile(&pFile); } void taos_error(TAOS_RES *tres, int64_t st) { @@ -969,7 +971,6 @@ void source_file(TAOS *con, char *fptr) { char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; - size_t line_len = 0; if (wordexp(fptr, &full_path, 0) != 0) { fprintf(stderr, "ERROR: illegal file name\n"); @@ -989,15 +990,16 @@ void source_file(TAOS *con, char *fptr) { } */ - FILE *f = fopen(fname, "r"); - if (f == NULL) { + // FILE *f = fopen(fname, "r"); + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { fprintf(stderr, "ERROR: failed to open file %s\n", fname); wordfree(&full_path); free(cmd); return; } - while ((read_len = tgetline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetLineFile(pFile, &line)) != -1) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; @@ -1020,9 +1022,9 @@ void source_file(TAOS *con, char *fptr) { } free(cmd); - if (line) free(line); + if(line != NULL) free(line); wordfree(&full_path); - fclose(f); + taosCloseFile(&pFile); } void shellGetGrantInfo(void *con) { diff --git a/tools/shell/src/shellLinux.c b/tools/shell/src/shellLinux.c index 81d8c0a35bf86acc47c5f040c18ac97a6d97d53c..57d6df00512014191d23fba41c4ee7c800556982 100644 --- a/tools/shell/src/shellLinux.c +++ b/tools/shell/src/shellLinux.c @@ -18,8 +18,9 @@ #include "tglobal.h" #include "shell.h" #include "shellCommand.h" -#include "tkey.h" -#include "ulog.h" +#include "tbase64.h" +#include "tlog.h" +#include "version.h" #include #include @@ -183,7 +184,6 @@ static void parse_args( for (int i = 1; i < argc; i++) { if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Linux"); printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 607dc792572ca07db43942d89b98f6d97e358eae..8a1763c4fc8d2f778bf6c0b2874ff16c5868cf48 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -15,7 +15,6 @@ #include "os.h" #include "shell.h" -#include "tconfig.h" #include "tglobal.h" pthread_t pid;